Variant shredding

parquet-avro/pom.xml

@@ -48,6 +48,11 @@
       <artifactId>parquet-common</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-variant</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>

parquet-avro/src/main/java/org/apache/parquet/avro/AvroConverters.java

@@ -31,8 +31,10 @@
 import org.apache.parquet.io.api.Binary;
 import org.apache.parquet.io.api.GroupConverter;
 import org.apache.parquet.io.api.PrimitiveConverter;
+import org.apache.parquet.schema.GroupType;
 import org.apache.parquet.schema.PrimitiveStringifier;
 import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.variant.VariantColumnConverter;
 
 public class AvroConverters {
 
@@ -363,4 +365,26 @@ public String convert(Binary binary) {
       return stringifier.stringify(binary);
     }
   }
+
+  static final class FieldVariantConverter<T> extends VariantColumnConverter {
+    protected final ParentValueContainer parent;
+    private final Schema avroSchema;
+    private final GenericData model;
+
+    public FieldVariantConverter(
+        ParentValueContainer parent, GroupType schema, Schema avroSchema, GenericData model) {
+      super(schema);
+      this.avroSchema = avroSchema;
+      this.model = model;
+      this.parent = parent;
+    }
+
+    @Override
+    public void addVariant(Binary value, Binary metadata) {
+      T currentRecord = (T) model.newRecord(null, avroSchema);
+      model.setField(currentRecord, "metadata", 0, metadata.toByteBuffer());
+      model.setField(currentRecord, "value", 1, value.toByteBuffer());
+      parent.add(currentRecord);
+    }
+  }
 }

parquet-avro/src/main/java/org/apache/parquet/avro/AvroIndexedRecordConverter.java

@@ -35,6 +35,7 @@
 import org.apache.parquet.io.api.GroupConverter;
 import org.apache.parquet.io.api.PrimitiveConverter;
 import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
 import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.Type;
 
@@ -168,7 +169,12 @@ private static Converter newConverter(Schema schema, Type type, GenericData mode
       case MAP:
         return new MapConverter(parent, type.asGroupType(), schema, model);
       case RECORD:
-        return new AvroIndexedRecordConverter<>(parent, type.asGroupType(), schema, model);
+        if (type.getLogicalTypeAnnotation()
+                instanceof LogicalTypeAnnotation.VariantLogicalTypeAnnotation) {
+          return new AvroConverters.FieldVariantConverter(parent, type.asGroupType(), schema, model);
+        } else {
+          return new AvroIndexedRecordConverter<>(parent, type.asGroupType(), schema, model);
+        }
       case STRING:
         return new AvroConverters.FieldStringConverter(parent);
       case UNION:

parquet-avro/src/main/java/org/apache/parquet/avro/AvroRecordConverter.java

@@ -62,6 +62,7 @@
 import org.apache.parquet.io.api.Converter;
 import org.apache.parquet.io.api.GroupConverter;
 import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
 import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.Type;
 import org.slf4j.Logger;
@@ -383,7 +384,12 @@ private static Converter newConverter(
         }
         return newStringConverter(schema, model, parent);
       case RECORD:
-        return new AvroRecordConverter(parent, type.asGroupType(), schema, model);
+        if (type.getLogicalTypeAnnotation()
+                instanceof LogicalTypeAnnotation.VariantLogicalTypeAnnotation) {
+          return new AvroConverters.FieldVariantConverter(parent, type.asGroupType(), schema, model);
+        } else {
+          return new AvroRecordConverter(parent, type.asGroupType(), schema, model);
+        }
       case ENUM:
         return new AvroConverters.FieldEnumConverter(parent, schema, model);
       case ARRAY:

parquet-avro/src/main/java/org/apache/parquet/avro/AvroSchemaConverter.java

@@ -466,6 +466,21 @@ public Optional<Schema> visit(
                   LogicalTypeAnnotation.EnumLogicalTypeAnnotation enumLogicalType) {
                 return of(Schema.create(Schema.Type.STRING));
               }
+
+              @Override
+              public Optional<Schema> visit(
+                  LogicalTypeAnnotation.VariantLogicalTypeAnnotation variantLogicalType) {
+                String name = parquetGroupType.getName();
+                SchemaBuilder.FieldAssembler<Schema> builder = SchemaBuilder.builder(
+                        namespace(name, names))
+                    .record(name)
+                    .fields();
+                builder.name("metadata")
+                    .type(Schema.create(Schema.Type.BYTES))
+                    .noDefault();
+                builder.name("value").type().optional().type(Schema.create(Schema.Type.BYTES));
+                return of(builder.endRecord());
+              }
             })
             .orElseThrow(
                 () -> new UnsupportedOperationException("Cannot convert Parquet type " + parquetType));