How to convert parquet schema to avro in Java/Scala

Question

Let say I have parquet file on the file system. How can I get parquet schema and convert it to Avro Schema?

Answer 1

Use hadoop ParquetFileReader to get Parquet schema and pass it to AvroSchemaConverter to convert it to Avro schema. Scala code example:

import org.apache.avro.Schema

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path

import org.apache.parquet.avro.AvroSchemaConverter
import org.apache.parquet.hadoop.ParquetFileReader
import org.apache.parquet.hadoop.util.HadoopInputFile

object ParquetToAvroSchemaConverter {
  def main(args: Array[String]): Unit = {
    val path = new Path("###PATH_TO_PARQUET_FILE###")
    val avroSchema = convert(path)
  }

  def convert(parquetPath: Path): Schema = {
    val cfg = new Configuration
    // Create parquet reader
    val rdr = ParquetFileReader.open(HadoopInputFile.fromPath(parquetPath, cfg))
    try {
      // Get parquet schema
      val schema = rdr.getFooter.getFileMetaData.getSchema
      println("Parquet schema: ")
      println("#############################################################")
      print(schema.toString)
      println("#############################################################")
      println

      // Convert to Avro
      val avroSchema = new AvroSchemaConverter(cfg).convert(schema)
      println("Avro schema: ")
      println("#############################################################")
      println(avroSchema.toString(true))
      println("#############################################################")

      avroSchema
    }
    finally {
      rdr.close()
    }
  }
}

You have to have next dependencies in your SBT project:

libraryDependencies ++= Seq(
  "org.apache.parquet" % "parquet-avro" % "1.10.0",
  "org.apache.parquet" % "parquet-hadoop" % "1.10.0",
  "org.apache.hadoop" % "hadoop-client" % "2.7.3",
)