org.apache.hadoop.hive.metastore.utils

Class MetaStoreServerUtils

java.lang.Object

org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils

public class MetaStoreServerUtils
extends Object

Utility methods used by Hive standalone metastore server.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	MetaStoreServerUtils.ColStatsObjWithSourceInfo

Field Summary

Fields

Modifier and Type	Field and Description
static String	JUNIT_DATABASE_PREFIX

Constructor Summary

Constructors

Constructor and Description
MetaStoreServerUtils()

Method Summary

Modifier and Type	Method and Description
static List<ColumnStatisticsObj>	aggrPartitionStats(List<ColumnStatistics> partStats, String catName, String dbName, String tableName, List<String> partNames, List<String> colNames, boolean areAllPartsFound, boolean useDensityFunctionForNDVEstimation, double ndvTuner)
static List<ColumnStatisticsObj>	aggrPartitionStats(Map<ColumnStatsAggregator,List<MetaStoreServerUtils.ColStatsObjWithSourceInfo>> colStatsMap, List<String> partNames, boolean areAllPartsFound, boolean useDensityFunctionForNDVEstimation, double ndvTuner)
static String	anonymizeConnectionURL(String connectionURL) Mask out all sensitive information from the jdbc connection url string.
static boolean	arePrefixColumns(List<FieldSchema> p, List<FieldSchema> s) Returns true if p is a prefix of s.
static boolean	areSameColumns(List<FieldSchema> oldCols, List<FieldSchema> newCols) Compare the names, types and comments of two lists of FieldSchema.
static boolean	checkUserHasHostProxyPrivileges(String user, org.apache.hadoop.conf.Configuration conf, String ipAddress) Verify if the user is allowed to make DB notification related calls.
static void	clearQuickStats(Map<String,String> params)
static boolean	columnsIncludedByNameType(List<FieldSchema> oldCols, List<FieldSchema> newCols)
static Partition	createMetaPartitionObject(Table tbl, Map<String,String> partSpec, org.apache.hadoop.fs.Path location)
static double	decimalToDouble(Decimal decimal)
static int	findFreePort() Finds a free port on the machine.
static int	findFreePortExcepting(int portToExclude) Finds a free port on the machine, but allow the ability to specify a port number to not use, no matter what.
static List<Partition>	getAllPartitionsOf(IMetaStoreClient msc, Table table)
static boolean	getBooleanEnvProp(EnvironmentContext envContext, String key)
static List<String>	getColumnNames(List<FieldSchema> cols) Generate list of lower case column names from the fieldSchema list
static org.apache.hadoop.fs.Path	getDataLocation(Table table, Partition partition)
static String	getIndexTableName(String dbName, String baseTblName, String indexName)
static void	getMergableCols(ColumnStatistics csNew, Map<String,String> parameters)
static <T> List<T>	getMetaStoreListeners(Class<T> clazz, org.apache.hadoop.conf.Configuration conf, String listenerImplList) create listener instances as per the configuration.
static String	getNormalisedPartitionValue(String partitionValue, String type)
static List<String>	getPartColNames(Table table)
static List<FieldSchema>	getPartCols(Table table)
static Partition	getPartition(IMetaStoreClient msc, Table tbl, Map<String,String> partSpec)
static Map<String,String>	getPartitionColtoTypeMap(List<FieldSchema> partitionCols)
static void	getPartitionListByFilterExp(IMetaStoreClient msc, Table table, byte[] filterExp, String defaultPartName, List<Partition> results)
static String	getPartitionName(org.apache.hadoop.fs.Path tablePath, org.apache.hadoop.fs.Path partitionPath, Set<String> partCols, Map<String,String> partitionColToTypeMap) Get the partition name from the path.
static String	getPartitionName(Table table, Partition partition)
static List<Partition>	getPartitionsByProjectSpec(IMetaStoreClient msc, GetPartitionsRequest request)
static Map<String,String>	getPartitionSpec(Table table, Partition partition)
static List<PartitionSpec>	getPartitionspecsGroupedByStorageDescriptor(Table table, Collection<Partition> partitions) Coalesce list of partitions belonging to a table into a more compact PartitionSpec representation.
static org.apache.hadoop.fs.Path	getPath(Table table)
static String	getTokenStoreClassName(org.apache.hadoop.conf.Configuration conf) This method should be used to return the metastore specific tokenstore class name to main backwards compatibility
static byte[]	hashStorageDescriptor(StorageDescriptor sd, MessageDigest md) Produce a hash for the storage descriptor
static boolean	isCompactionTxn(TxnType txnType)
static boolean	isFastStatsSame(Partition oldPart, Partition newPart) Determines whether the "fast stats" for the passed partitions are the same.
static boolean	isPartitioned(Table table)
static boolean	isTransactionalTable(Map<String,String> params) Duplicates AcidUtils; used in a couple places in metastore.
static void	mergeColStats(ColumnStatistics csNew, ColumnStatistics csOld)
static boolean	partitionNameHasValidCharacters(List<String> partVals, Pattern partitionValidationPattern)
static void	populateQuickStats(List<org.apache.hadoop.fs.FileStatus> fileStatus, Map<String,String> params) This method is invalid for MM and ACID tables unless fileStatus comes from AcidUtils.
static boolean	requireCalStats(Partition oldPart, Partition newPart, Table tbl, EnvironmentContext environmentContext)
static void	setNestedProperty(Object bean, String propertyName, Object value, boolean instantiateMissingFields) This is a util method to set a nested property of a given object.
static int	startMetaStore()
static int	startMetaStore(org.apache.hadoop.conf.Configuration conf)
static int	startMetaStore(HadoopThriftAuthBridge bridge, org.apache.hadoop.conf.Configuration conf)
static void	startMetaStore(int port, HadoopThriftAuthBridge bridge)
static void	startMetaStore(int port, HadoopThriftAuthBridge bridge, org.apache.hadoop.conf.Configuration hiveConf)
static Map<String,String>	trimMapNulls(Map<String,String> dnMap, boolean retrieveMapNullsAsEmptyStrings) We have a need to sanity-check the map before conversion from persisted objects to metadata thrift objects because null values in maps will cause a NPE if we send across thrift.
static void	updateBasicState(EnvironmentContext environmentContext, Map<String,String> params)
static boolean	updatePartitionStatsFast(PartitionSpecProxy.PartitionIterator part, Table table, Warehouse wh, boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext, boolean isCreate) Updates the numFiles and totalSize parameters for the passed Partition by querying the warehouse if the passed Partition does not already have values for these parameters.
static boolean	updatePartitionStatsFast(Partition part, Table tbl, Warehouse wh, boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext, boolean isCreate) Updates the numFiles and totalSize parameters for the passed Partition by querying the warehouse if the passed Partition does not already have values for these parameters.
static void	updateTableStatsForCreateTable(Warehouse wh, Database db, Table tbl, EnvironmentContext envContext, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path tblPath, boolean newDir)
static void	updateTableStatsSlow(Database db, Table tbl, Warehouse wh, boolean newDir, boolean forceRecompute, EnvironmentContext environmentContext) Updates the numFiles and totalSize parameters for the passed Table by querying the warehouse if the passed Table does not already have values for these parameters.
static List<String>	validateBucketColumns(StorageDescriptor sd) Validate bucket columns should belong to table columns.
static boolean	validateColumnName(String name)
static void	validatePartitionNameCharacters(List<String> partVals, Pattern partitionValidationPattern)
static String	validateSkewedColNames(List<String> cols)
static String	validateSkewedColNamesSubsetCol(List<String> skewedColNames, List<FieldSchema> cols)
static String	validateTblColumns(List<FieldSchema> cols)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

JUNIT_DATABASE_PREFIX

public static final String JUNIT_DATABASE_PREFIX

See Also:

Constant Field Values

Constructor Detail

MetaStoreServerUtils

public MetaStoreServerUtils()

Method Detail

trimMapNulls

public static Map<String,String> trimMapNulls(Map<String,String> dnMap,
                                              boolean retrieveMapNullsAsEmptyStrings)

We have a need to sanity-check the map before conversion from persisted objects to metadata thrift objects because null values in maps will cause a NPE if we send across thrift. Pruning is appropriate for most cases except for databases such as Oracle where Empty strings are stored as nulls, in which case we need to handle that. See HIVE-8485 for motivations for this.

aggrPartitionStats

public static List<ColumnStatisticsObj> aggrPartitionStats(List<ColumnStatistics> partStats,
                                                           String catName,
                                                           String dbName,
                                                           String tableName,
                                                           List<String> partNames,
                                                           List<String> colNames,
                                                           boolean areAllPartsFound,
                                                           boolean useDensityFunctionForNDVEstimation,
                                                           double ndvTuner)
                                                    throws MetaException

Throws:

MetaException

aggrPartitionStats

public static List<ColumnStatisticsObj> aggrPartitionStats(Map<ColumnStatsAggregator,List<MetaStoreServerUtils.ColStatsObjWithSourceInfo>> colStatsMap,
                                                           List<String> partNames,
                                                           boolean areAllPartsFound,
                                                           boolean useDensityFunctionForNDVEstimation,
                                                           double ndvTuner)
                                                    throws MetaException

Throws:

MetaException

decimalToDouble

public static double decimalToDouble(Decimal decimal)

validatePartitionNameCharacters

public static void validatePartitionNameCharacters(List<String> partVals,
                                                   Pattern partitionValidationPattern)
                                            throws MetaException

Throws:

MetaException

hashStorageDescriptor

public static byte[] hashStorageDescriptor(StorageDescriptor sd,
                                           MessageDigest md)

Produce a hash for the storage descriptor

Parameters:

sd - storage descriptor to hash

md - message descriptor to use to generate the hash

Returns:

the hash as a byte array

validateColumnName

public static boolean validateColumnName(String name)

isFastStatsSame

public static boolean isFastStatsSame(Partition oldPart,
                                      Partition newPart)

Determines whether the "fast stats" for the passed partitions are the same.

Parameters:

oldPart - Old partition to compare.

newPart - New partition to compare.

Returns:

true if the partitions are not null, contain all the "fast stats" and have the same values for these stats, otherwise false.

updateTableStatsSlow

public static void updateTableStatsSlow(Database db,
                                        Table tbl,
                                        Warehouse wh,
                                        boolean newDir,
                                        boolean forceRecompute,
                                        EnvironmentContext environmentContext)
                                 throws MetaException

Updates the numFiles and totalSize parameters for the passed Table by querying the warehouse if the passed Table does not already have values for these parameters. NOTE: This function is rather expensive since it needs to traverse the file system to get all the information.

Parameters:

newDir - if true, the directory was just created and can be assumed to be empty

forceRecompute - Recompute stats even if the passed Table already has these parameters set

Throws:

MetaException

populateQuickStats

public static void populateQuickStats(List<org.apache.hadoop.fs.FileStatus> fileStatus,
                                      Map<String,String> params)

This method is invalid for MM and ACID tables unless fileStatus comes from AcidUtils.

clearQuickStats

public static void clearQuickStats(Map<String,String> params)

updateTableStatsForCreateTable

public static void updateTableStatsForCreateTable(Warehouse wh,
                                                  Database db,
                                                  Table tbl,
                                                  EnvironmentContext envContext,
                                                  org.apache.hadoop.conf.Configuration conf,
                                                  org.apache.hadoop.fs.Path tblPath,
                                                  boolean newDir)
                                           throws MetaException

Throws:

MetaException

getBooleanEnvProp

public static boolean getBooleanEnvProp(EnvironmentContext envContext,
                                        String key)

areSameColumns

public static boolean areSameColumns(List<FieldSchema> oldCols,
                                     List<FieldSchema> newCols)

Compare the names, types and comments of two lists of FieldSchema.

The name of FieldSchema is compared in the case-insensitive mode because all names in Hive are case-insensitive.

Parameters:

oldCols - old columns

newCols - new columns

Returns:

true if the two columns are the same, false otherwise

arePrefixColumns

public static boolean arePrefixColumns(List<FieldSchema> p,
                                       List<FieldSchema> s)

Returns true if p is a prefix of s.

The compare of FieldSchema is the same as areSameColumns(List, List).

updateBasicState

public static void updateBasicState(EnvironmentContext environmentContext,
                                    Map<String,String> params)

updatePartitionStatsFast

public static boolean updatePartitionStatsFast(Partition part,
                                               Table tbl,
                                               Warehouse wh,
                                               boolean madeDir,
                                               boolean forceRecompute,
                                               EnvironmentContext environmentContext,
                                               boolean isCreate)
                                        throws MetaException

Updates the numFiles and totalSize parameters for the passed Partition by querying the warehouse if the passed Partition does not already have values for these parameters.

Parameters:

part -

wh -

madeDir - if true, the directory was just created and can be assumed to be empty

forceRecompute - Recompute stats even if the passed Partition already has these parameters set

Returns:

true if the stats were updated, false otherwise

Throws:

MetaException

updatePartitionStatsFast

public static boolean updatePartitionStatsFast(PartitionSpecProxy.PartitionIterator part,
                                               Table table,
                                               Warehouse wh,
                                               boolean madeDir,
                                               boolean forceRecompute,
                                               EnvironmentContext environmentContext,
                                               boolean isCreate)
                                        throws MetaException

Updates the numFiles and totalSize parameters for the passed Partition by querying the warehouse if the passed Partition does not already have values for these parameters.

Parameters:

part -

wh -

madeDir - if true, the directory was just created and can be assumed to be empty

forceRecompute - Recompute stats even if the passed Partition already has these parameters set

Returns:

true if the stats were updated, false otherwise

Throws:

MetaException

columnsIncludedByNameType

public static boolean columnsIncludedByNameType(List<FieldSchema> oldCols,
                                                List<FieldSchema> newCols)

isTransactionalTable

public static boolean isTransactionalTable(Map<String,String> params)

Duplicates AcidUtils; used in a couple places in metastore.

getMetaStoreListeners

public static <T> List<T> getMetaStoreListeners(Class<T> clazz,
                                                org.apache.hadoop.conf.Configuration conf,
                                                String listenerImplList)
                                         throws MetaException

create listener instances as per the configuration.

Parameters:

clazz - Class of the listener

conf - configuration object

listenerImplList - Implementation class name

Returns:

instance of the listener

Throws:

MetaException - if there is any failure instantiating the class

validateSkewedColNames

public static String validateSkewedColNames(List<String> cols)

validateSkewedColNamesSubsetCol

public static String validateSkewedColNamesSubsetCol(List<String> skewedColNames,
                                                     List<FieldSchema> cols)

partitionNameHasValidCharacters

public static boolean partitionNameHasValidCharacters(List<String> partVals,
                                                      Pattern partitionValidationPattern)

getMergableCols

public static void getMergableCols(ColumnStatistics csNew,
                                   Map<String,String> parameters)

mergeColStats

public static void mergeColStats(ColumnStatistics csNew,
                                 ColumnStatistics csOld)
                          throws InvalidObjectException

Throws:

InvalidObjectException

checkUserHasHostProxyPrivileges

public static boolean checkUserHasHostProxyPrivileges(String user,
                                                      org.apache.hadoop.conf.Configuration conf,
                                                      String ipAddress)

Verify if the user is allowed to make DB notification related calls. Only the superusers defined in the Hadoop proxy user settings have the permission.

Parameters:

user - the short user name

conf - that contains the proxy user settings

Returns:

if the user has the permission

startMetaStore

public static int startMetaStore()
                          throws Exception

Throws:

Exception

startMetaStore

public static int startMetaStore(HadoopThriftAuthBridge bridge,
                                 org.apache.hadoop.conf.Configuration conf)
                          throws Exception

Throws:

Exception

startMetaStore

public static int startMetaStore(org.apache.hadoop.conf.Configuration conf)
                          throws Exception

Throws:

Exception

startMetaStore

public static void startMetaStore(int port,
                                  HadoopThriftAuthBridge bridge)
                           throws Exception

Throws:

Exception

startMetaStore

public static void startMetaStore(int port,
                                  HadoopThriftAuthBridge bridge,
                                  org.apache.hadoop.conf.Configuration hiveConf)
                           throws Exception

Throws:

Exception

findFreePort

public static int findFreePort()
                        throws IOException

Finds a free port on the machine.

Returns:

Throws:

IOException

findFreePortExcepting

public static int findFreePortExcepting(int portToExclude)
                                 throws IOException

Finds a free port on the machine, but allow the ability to specify a port number to not use, no matter what.

Throws:

IOException

getIndexTableName

public static String getIndexTableName(String dbName,
                                       String baseTblName,
                                       String indexName)

validateTblColumns

public static String validateTblColumns(List<FieldSchema> cols)

requireCalStats

public static boolean requireCalStats(Partition oldPart,
                                      Partition newPart,
                                      Table tbl,
                                      EnvironmentContext environmentContext)

getTokenStoreClassName

public static String getTokenStoreClassName(org.apache.hadoop.conf.Configuration conf)

This method should be used to return the metastore specific tokenstore class name to main backwards compatibility

Parameters:

conf - - HiveConf object

Returns:

the tokenStoreClass name from the HiveConf. It maps the hive specific tokenstoreclass name to metastore module specific class name. For eg: hive.cluster.delegation.token.store.class is set to org.apache.hadoop.hive.thrift.MemoryTokenStore it returns the equivalent tokenstore class defined in the metastore module which is org.apache.hadoop.hive.metastore.security.MemoryTokenStore Similarly, org.apache.hadoop.hive.thrift.DBTokenStore maps to org.apache.hadoop.hive.metastore.security.DBTokenStore and org.apache.hadoop.hive.thrift.ZooKeeperTokenStore maps to org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore

getPartitionspecsGroupedByStorageDescriptor

public static List<PartitionSpec> getPartitionspecsGroupedByStorageDescriptor(Table table,
                                                                              Collection<Partition> partitions)

Coalesce list of partitions belonging to a table into a more compact PartitionSpec representation.

Parameters:

table - Table thrift object

partitions - List of partition objects

Returns:

collection PartitionSpec objects which is a compressed representation of original partition list.

setNestedProperty

public static void setNestedProperty(Object bean,
                                     String propertyName,
                                     Object value,
                                     boolean instantiateMissingFields)
                              throws MetaException

This is a util method to set a nested property of a given object. The nested property is a dot separated string where each nesting level is separated by a dot. This method makes use of PropertyUtils methods from apache-commons library and assumes that the field names provided in the input propertyName have valid setters. eg. the propertyName sd.serdeInfo.inputFormat represents the inputformat field of the serdeInfo field of the sd field. The argument bean should have these fields (in this case it should be a Partition object). The value argument is the value to be set for the nested field. Note that if in case of one of nested levels is null you must set instantiateMissingFields argument to true otherwise this method could throw a NPE.

Parameters:

bean - the object whose nested field needs to be set. This object must have setter methods defined for each nested field name in the propertyName

propertyName - the nested propertyName to be set. Each level of nesting is dot separated

value - the value to which the nested property is set

instantiateMissingFields - in case of some nestedFields being nulls, setting this argument to true will attempt to instantiate the missing fields using the default constructor. If there is no default constructor available this would throw a MetaException

Throws:

MetaException

anonymizeConnectionURL

public static String anonymizeConnectionURL(String connectionURL)

Mask out all sensitive information from the jdbc connection url string.

Parameters:

connectionURL - the connection url, can be null

Returns:

the anonymized connection url , can be null

getPath

public static org.apache.hadoop.fs.Path getPath(Table table)

getAllPartitionsOf

public static List<Partition> getAllPartitionsOf(IMetaStoreClient msc,
                                                 Table table)
                                          throws MetastoreException

Throws:

MetastoreException

getPartitionsByProjectSpec

public static List<Partition> getPartitionsByProjectSpec(IMetaStoreClient msc,
                                                         GetPartitionsRequest request)
                                                  throws MetastoreException

Throws:

MetastoreException

getPartitionListByFilterExp

public static void getPartitionListByFilterExp(IMetaStoreClient msc,
                                               Table table,
                                               byte[] filterExp,
                                               String defaultPartName,
                                               List<Partition> results)
                                        throws MetastoreException

Throws:

MetastoreException

isPartitioned

public static boolean isPartitioned(Table table)

getPartCols

public static List<FieldSchema> getPartCols(Table table)

getPartColNames

public static List<String> getPartColNames(Table table)

getDataLocation

public static org.apache.hadoop.fs.Path getDataLocation(Table table,
                                                        Partition partition)

getPartitionName

public static String getPartitionName(Table table,
                                      Partition partition)

getPartitionSpec

public static Map<String,String> getPartitionSpec(Table table,
                                                  Partition partition)

getPartition

public static Partition getPartition(IMetaStoreClient msc,
                                     Table tbl,
                                     Map<String,String> partSpec)
                              throws MetastoreException

Throws:

MetastoreException

getPartitionName

public static String getPartitionName(org.apache.hadoop.fs.Path tablePath,
                                      org.apache.hadoop.fs.Path partitionPath,
                                      Set<String> partCols,
                                      Map<String,String> partitionColToTypeMap)

Get the partition name from the path.

Parameters:

tablePath - Path of the table.

partitionPath - Path of the partition.

partCols - Set of partition columns from table definition

Returns:

Partition name, for example partitiondate=2008-01-01

getNormalisedPartitionValue

public static String getNormalisedPartitionValue(String partitionValue,
                                                 String type)

getPartitionColtoTypeMap

public static Map<String,String> getPartitionColtoTypeMap(List<FieldSchema> partitionCols)

createMetaPartitionObject

public static Partition createMetaPartitionObject(Table tbl,
                                                  Map<String,String> partSpec,
                                                  org.apache.hadoop.fs.Path location)
                                           throws MetastoreException

Throws:

MetastoreException

validateBucketColumns

public static List<String> validateBucketColumns(StorageDescriptor sd)

Validate bucket columns should belong to table columns.

Parameters:

sd - StorageDescriptor of given table

Returns:

true if bucket columns are empty or belong to table columns else false

getColumnNames

public static List<String> getColumnNames(List<FieldSchema> cols)

Generate list of lower case column names from the fieldSchema list

Parameters:

cols - fieldSchema list

Returns:

column name list

isCompactionTxn

public static boolean isCompactionTxn(TxnType txnType)