org.apache.hudi.cli.commands

Class MetadataCommand

java.lang.Object

org.apache.hudi.cli.commands.MetadataCommand

@ShellComponent
public class MetadataCommand
extends Object

CLI commands to operate on the Metadata Table.

Example: The default spark.master conf is set to yarn. If you are running on a local deployment, we can set the spark master to local using set conf command. > set --conf SPARK_MASTER=local[2]

Connect to the table > connect --path {path to hudi table}

Run metadata commands > metadata list-partitions

Constructor Summary

Constructors

Constructor and Description
MetadataCommand()

Method Summary

Modifier and Type	Method and Description
String	create(String master)
String	delete(boolean backup)
String	deleteRecordIndex(boolean backup)
static String	getMetadataTableBasePath(String tableBasePath)
String	init(String master, boolean readOnly)
String	listFiles(String partition)
String	listPartitions(String master)
String	set(String metadataDir)
static void	setMetadataBaseDirectory(String metadataDir) Sets the directory to store/read Metadata Table.
String	stats()
String	validateFiles(boolean verbose)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

MetadataCommand

public MetadataCommand()

Method Detail

setMetadataBaseDirectory

public static void setMetadataBaseDirectory(String metadataDir)

Sets the directory to store/read Metadata Table.

This can be used to store the metadata table away from the dataset directory. - Useful for testing as well as for using via the HUDI CLI so that the actual dataset is not written to. - Useful for testing Metadata Table performance and operations on existing datasets before enabling.

getMetadataTableBasePath

public static String getMetadataTableBasePath(String tableBasePath)

set

@ShellMethod(key="metadata set",
             value="Set options for Metadata Table")
public String set(@ShellOption(value="--metadataDir",help="Directory to read/write metadata table (can be different from dataset)",defaultValue="")
                                                                                                        String metadataDir)

create

@ShellMethod(key="metadata create",
             value="Create the Metadata Table if it does not exist")
public String create(@ShellOption(value="--sparkMaster",defaultValue="yarn",help="Spark master")
                                                                                                                              String master)
                                                                                                                       throws Exception

Throws:

Exception

delete

@ShellMethod(key="metadata delete",
             value="Remove the Metadata Table")
public String delete(@ShellOption(value="--backup",help="Backup the metadata table before delete",defaultValue="true",arity=1)
                                                                                                         boolean backup)
                                                                                                  throws Exception

Throws:

Exception

deleteRecordIndex

@ShellMethod(key="metadata delete-record-index",
             value="Delete the record index from Metadata Table")
public String deleteRecordIndex(@ShellOption(value="--backup",help="Backup the record index before delete",defaultValue="true",arity=1)
                                                                                                                                                   boolean backup)
                                                                                                                                            throws Exception

Throws:

Exception

init

@ShellMethod(key="metadata init",
             value="Update the metadata table from commits since the creation")
public String init(@ShellOption(value="--sparkMaster",defaultValue="yarn",help="Spark master")
                                                                                                                                     String master,
                                                                                                                                     @ShellOption(value="--readonly",defaultValue="false",help="Open in read-only mode")
                                                                                                                                     boolean readOnly)
                                                                                                                              throws Exception

Throws:

Exception

stats

@ShellMethod(key="metadata stats",
             value="Print stats about the metadata")
public String stats()
                                                                                                     throws IOException

Throws:

IOException

listPartitions

@ShellMethod(key="metadata list-partitions",
             value="List all partitions from metadata")
public String listPartitions(@ShellOption(value="--sparkMaster",defaultValue="yarn",help="Spark master")
                                                                                                                                  String master)
                                                                                                                           throws IOException

Throws:

IOException

listFiles

@ShellMethod(key="metadata list-files",
             value="Print a list of all files in a partition from the metadata")
public String listFiles(@ShellOption(value="--partition",help="Name of the partition to list files",defaultValue="")
                                                                                                                                                 String partition)
                                                                                                                                          throws IOException

Throws:

IOException

validateFiles

@ShellMethod(key="metadata validate-files",
             value="Validate all files in all partitions from the metadata")
public String validateFiles(@ShellOption(value="--verbose",help="Print all file details",defaultValue="false")
                                                                                                                                                     boolean verbose)
                                                                                                                                              throws IOException

Throws:

IOException