org.apache.hadoop.hive.ql.io.orc

Class NiFiOrcUtils

java.lang.Object

org.apache.hadoop.hive.ql.io.orc.NiFiOrcUtils

public class NiFiOrcUtils
extends Object

Utility methods for ORC support (conversion from Avro, conversion to Hive types, e.g.

Field Summary

Fields

Modifier and Type	Field and Description
private static org.apache.hadoop.hive.ql.io.orc.MemoryManager	memoryManager

Constructor Summary

Constructors

Constructor and Description
NiFiOrcUtils()

Method Summary

Modifier and Type	Method and Description
static Object	convertToORCObject(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo typeInfo, Object o)
static org.apache.hadoop.hive.ql.io.orc.OrcStruct	createOrcStruct(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo typeInfo, Object... objs) Create an object of OrcStruct given a TypeInfo and a list of objects
static OrcFlowFileWriter	createWriter(OutputStream flowFileOutputStream, org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hive.serde2.typeinfo.TypeInfo orcSchema, long stripeSize, org.apache.hadoop.hive.ql.io.orc.CompressionKind compress, int bufferSize)
static String	generateHiveDDL(org.apache.avro.Schema avroSchema, String tableName)
static org.apache.avro.Schema.Type	getAvroSchemaTypeOfObject(Object o)
static String	getHiveTypeFromAvroType(org.apache.avro.Schema avroSchema)
static String	getHiveTypeFromFieldType(DataType rawDataType, boolean hiveFieldNames)
private static org.apache.hadoop.hive.serde2.typeinfo.TypeInfo	getLogicalTypeInfo(org.apache.avro.Schema schema)
private static org.apache.hadoop.hive.ql.io.orc.MemoryManager	getMemoryManager(org.apache.hadoop.conf.Configuration conf)
static org.apache.hadoop.hive.serde2.typeinfo.TypeInfo	getOrcField(org.apache.avro.Schema fieldSchema)
static org.apache.hadoop.hive.serde2.typeinfo.TypeInfo	getPrimitiveOrcTypeFromPrimitiveAvroType(org.apache.avro.Schema.Type avroType)
private static boolean	isLogicalType(org.apache.avro.Schema schema)
static String	normalizeHiveTableName(String name)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

memoryManager

private static org.apache.hadoop.hive.ql.io.orc.MemoryManager memoryManager

Constructor Detail

NiFiOrcUtils

public NiFiOrcUtils()

Method Detail

convertToORCObject

public static Object convertToORCObject(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo typeInfo,
                                        Object o)

createOrcStruct

public static org.apache.hadoop.hive.ql.io.orc.OrcStruct createOrcStruct(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo typeInfo,
                                                                         Object... objs)

Create an object of OrcStruct given a TypeInfo and a list of objects

Parameters:

typeInfo - The TypeInfo object representing the ORC record schema

objs - ORC objects/Writables

Returns:

an OrcStruct containing the specified objects for the specified schema

normalizeHiveTableName

public static String normalizeHiveTableName(String name)

generateHiveDDL

public static String generateHiveDDL(org.apache.avro.Schema avroSchema,
                                     String tableName)

getOrcField

public static org.apache.hadoop.hive.serde2.typeinfo.TypeInfo getOrcField(org.apache.avro.Schema fieldSchema)
                                                                   throws IllegalArgumentException

Throws:

IllegalArgumentException

isLogicalType

private static boolean isLogicalType(org.apache.avro.Schema schema)

getLogicalTypeInfo

private static org.apache.hadoop.hive.serde2.typeinfo.TypeInfo getLogicalTypeInfo(org.apache.avro.Schema schema)

getAvroSchemaTypeOfObject

public static org.apache.avro.Schema.Type getAvroSchemaTypeOfObject(Object o)

getPrimitiveOrcTypeFromPrimitiveAvroType

public static org.apache.hadoop.hive.serde2.typeinfo.TypeInfo getPrimitiveOrcTypeFromPrimitiveAvroType(org.apache.avro.Schema.Type avroType)
                                                                                                throws IllegalArgumentException

Throws:

IllegalArgumentException

getHiveTypeFromAvroType

public static String getHiveTypeFromAvroType(org.apache.avro.Schema avroSchema)

getHiveTypeFromFieldType

public static String getHiveTypeFromFieldType(DataType rawDataType,
                                              boolean hiveFieldNames)

createWriter

public static OrcFlowFileWriter createWriter(OutputStream flowFileOutputStream,
                                             org.apache.hadoop.fs.Path path,
                                             org.apache.hadoop.conf.Configuration conf,
                                             org.apache.hadoop.hive.serde2.typeinfo.TypeInfo orcSchema,
                                             long stripeSize,
                                             org.apache.hadoop.hive.ql.io.orc.CompressionKind compress,
                                             int bufferSize)
                                      throws IOException

Throws:

IOException

getMemoryManager

private static org.apache.hadoop.hive.ql.io.orc.MemoryManager getMemoryManager(org.apache.hadoop.conf.Configuration conf)