org.apache.hudi.aws.sync

Class AWSGlueCatalogSyncClient

java.lang.Object

org.apache.hudi.sync.common.HoodieSyncClient

org.apache.hudi.aws.sync.AWSGlueCatalogSyncClient

All Implemented Interfaces:

AutoCloseable, org.apache.hudi.sync.common.HoodieMetaSyncOperations

public class AWSGlueCatalogSyncClient
extends org.apache.hudi.sync.common.HoodieSyncClient

This class implements all the AWS APIs to enable syncing of a Hudi Table with the AWS Glue Data Catalog (https://docs.aws.amazon.com/glue/latest/dg/populate-data-catalog.html).

Field Summary

Fields

Modifier and Type	Field and Description
protected software.amazon.awssdk.services.glue.GlueAsyncClient	awsGlue

Fields inherited from class org.apache.hudi.sync.common.HoodieSyncClient

config, metaClient, partitionValueExtractor

Fields inherited from interface org.apache.hudi.sync.common.HoodieMetaSyncOperations

HOODIE_LAST_COMMIT_COMPLETION_TIME_SYNC, HOODIE_LAST_COMMIT_TIME_SYNC

Constructor Summary

Constructors

Constructor and Description
AWSGlueCatalogSyncClient(HiveSyncConfig config)

Method Summary

Modifier and Type	Method and Description
void	addPartitionsToTable(String tableName, List<String> partitionsToAdd)
void	close()
void	createDatabase(String databaseName)
void	createOrReplaceTable(String tableName, org.apache.parquet.schema.MessageType storageSchema, String inputFormatClass, String outputFormatClass, String serdeClass, Map<String,String> serdeProperties, Map<String,String> tableProperties)
void	createTable(String tableName, org.apache.parquet.schema.MessageType storageSchema, String inputFormatClass, String outputFormatClass, String serdeClass, Map<String,String> serdeProperties, Map<String,String> tableProperties)
boolean	databaseExists(String databaseName)
void	deleteLastReplicatedTimeStamp(String tableName)
void	dropPartitions(String tableName, List<String> partitionsToDrop)
void	dropTable(String tableName)
String	generatePushDownFilter(List<String> writtenPartitions, List<org.apache.hudi.sync.common.model.FieldSchema> partitionFields)
List<org.apache.hudi.sync.common.model.Partition>	getAllPartitions(String tableName)
Option<String>	getLastCommitTimeSynced(String tableName)
Option<String>	getLastReplicatedTime(String tableName)
List<org.apache.hudi.sync.common.model.FieldSchema>	getMetastoreFieldSchemas(String tableName)
Map<String,String>	getMetastoreSchema(String tableName)
Boolean	getPartitionIndexEnable(String tableName)
List<org.apache.hudi.sync.common.model.Partition>	getPartitionsFromList(String tableName, List<String> partitionList)
List<org.apache.hudi.sync.common.model.FieldSchema>	getStorageFieldSchemas()
String	getTableLocation(String tableName)
void	managePartitionIndexes(String tableName) This will manage partitions indexes.
protected List<List<String>>	parsePartitionsIndexConfig()
boolean	tableExists(String tableName)
void	updateLastCommitTimeSynced(String tableName)
void	updateLastReplicatedTimeStamp(String tableName, String timeStamp)
void	updatePartitionIndexEnable(String tableName, Boolean enable)
void	updatePartitionsToTable(String tableName, List<String> changedPartitions)
boolean	updateSerdeProperties(String tableName, Map<String,String> serdeProperties, boolean useRealtimeFormat)
boolean	updateTableComments(String tableName, List<org.apache.hudi.sync.common.model.FieldSchema> fromMetastore, List<org.apache.hudi.sync.common.model.FieldSchema> fromStorage)
boolean	updateTableProperties(String tableName, Map<String,String> tableProperties)
void	updateTableSchema(String tableName, org.apache.parquet.schema.MessageType newSchema)

Methods inherited from class org.apache.hudi.sync.common.HoodieSyncClient

generateTempTableName, getActiveTimeline, getAllPartitionPathsOnStorage, getBasePath, getDroppedPartitionsSince, getMetaClient, getPartitionEvents, getPartitionEvents, getStorageSchema, getStorageSchema, getTableType, getWrittenPartitionsSince, isBootstrap

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.hudi.sync.common.HoodieMetaSyncOperations

getLastCommitCompletionTimeSynced

Field Detail

awsGlue

protected final software.amazon.awssdk.services.glue.GlueAsyncClient awsGlue

Constructor Detail

AWSGlueCatalogSyncClient

public AWSGlueCatalogSyncClient(HiveSyncConfig config)

Method Detail

getAllPartitions

public List<org.apache.hudi.sync.common.model.Partition> getAllPartitions(String tableName)

getPartitionsFromList

public List<org.apache.hudi.sync.common.model.Partition> getPartitionsFromList(String tableName,
                                                                               List<String> partitionList)

addPartitionsToTable

public void addPartitionsToTable(String tableName,
                                 List<String> partitionsToAdd)

updatePartitionsToTable

public void updatePartitionsToTable(String tableName,
                                    List<String> changedPartitions)

dropPartitions

public void dropPartitions(String tableName,
                           List<String> partitionsToDrop)

updateTableProperties

public boolean updateTableProperties(String tableName,
                                     Map<String,String> tableProperties)

getStorageFieldSchemas

public List<org.apache.hudi.sync.common.model.FieldSchema> getStorageFieldSchemas()

updateTableComments

public boolean updateTableComments(String tableName,
                                   List<org.apache.hudi.sync.common.model.FieldSchema> fromMetastore,
                                   List<org.apache.hudi.sync.common.model.FieldSchema> fromStorage)

updateTableSchema

public void updateTableSchema(String tableName,
                              org.apache.parquet.schema.MessageType newSchema)

createOrReplaceTable

public void createOrReplaceTable(String tableName,
                                 org.apache.parquet.schema.MessageType storageSchema,
                                 String inputFormatClass,
                                 String outputFormatClass,
                                 String serdeClass,
                                 Map<String,String> serdeProperties,
                                 Map<String,String> tableProperties)

createTable

public void createTable(String tableName,
                        org.apache.parquet.schema.MessageType storageSchema,
                        String inputFormatClass,
                        String outputFormatClass,
                        String serdeClass,
                        Map<String,String> serdeProperties,
                        Map<String,String> tableProperties)

managePartitionIndexes

public void managePartitionIndexes(String tableName)
                            throws ExecutionException,
                                   InterruptedException

This will manage partitions indexes. Users can activate/deactivate them on existing tables. Removing index definition, will result in dropping the index.

reference doc for partition indexes: https://docs.aws.amazon.com/glue/latest/dg/partition-indexes.html#partition-index-getpartitions

Parameters:

tableName -

Throws:

ExecutionException

InterruptedException

parsePartitionsIndexConfig

protected List<List<String>> parsePartitionsIndexConfig()

getPartitionIndexEnable

public Boolean getPartitionIndexEnable(String tableName)

updatePartitionIndexEnable

public void updatePartitionIndexEnable(String tableName,
                                       Boolean enable)

getMetastoreSchema

public Map<String,String> getMetastoreSchema(String tableName)

tableExists

public boolean tableExists(String tableName)

databaseExists

public boolean databaseExists(String databaseName)

createDatabase

public void createDatabase(String databaseName)

getLastCommitTimeSynced

public Option<String> getLastCommitTimeSynced(String tableName)

close

public void close()

updateLastCommitTimeSynced

public void updateLastCommitTimeSynced(String tableName)

dropTable

public void dropTable(String tableName)

getMetastoreFieldSchemas

public List<org.apache.hudi.sync.common.model.FieldSchema> getMetastoreFieldSchemas(String tableName)

getLastReplicatedTime

public Option<String> getLastReplicatedTime(String tableName)

updateLastReplicatedTimeStamp

public void updateLastReplicatedTimeStamp(String tableName,
                                          String timeStamp)

deleteLastReplicatedTimeStamp

public void deleteLastReplicatedTimeStamp(String tableName)

generatePushDownFilter

public String generatePushDownFilter(List<String> writtenPartitions,
                                     List<org.apache.hudi.sync.common.model.FieldSchema> partitionFields)

updateSerdeProperties

public boolean updateSerdeProperties(String tableName,
                                     Map<String,String> serdeProperties,
                                     boolean useRealtimeFormat)

getTableLocation

public String getTableLocation(String tableName)