iceberg 源码之FindFiles
文章目录
@Test
public void testWithMetadataMatching() {
table.newAppend()
.appendFile(FILE_A)
.appendFile(FILE_B)
.appendFile(FILE_C)
.appendFile(FILE_D)
.commit();
Iterable<DataFile> files = FindFiles.in(table)
.withMetadataMatching(Expressions.startsWith("file_path", "/path/to/data-a"))
.collect();
Assert.assertEquals(pathSet(FILE_A), pathSet(files));
}这里使用的file_path是一个iceberg表的元数据字段,在后台会翻译成字段信息出来。
Evaluator evaluator;
if (fileFilter != null && fileFilter != Expressions.alwaysTrue()) {
evaluator = new Evaluator(DataFile.getType(EMPTY_STRUCT), fileFilter, caseSensitive);
} else {
evaluator = null;
}其中evaluator如下所示:
DataFile的元数据信息如下:
// fields for adding delete data files
Types.NestedField CONTENT = optional(134, "content", IntegerType.get(),
"Contents of the file: 0=data, 1=position deletes, 2=equality deletes");
Types.NestedField FILE_PATH = required(100, "file_path", StringType.get(), "Location URI with FS scheme");
Types.NestedField FILE_FORMAT = required(101, "file_format", StringType.get(),
"File format name: avro, orc, or parquet");
Types.NestedField RECORD_COUNT = required(103, "record_count", LongType.get(), "Number of records in the file");
Types.NestedField FILE_SIZE = required(104, "file_size_in_bytes", LongType.get(), "Total file size in bytes");
Types.NestedField COLUMN_SIZES = optional(108, "column_sizes", MapType.ofRequired(117, 118,
IntegerType.get(), LongType.get()), "Map of column id to total size on disk");
Types.NestedField VALUE_COUNTS = optional(109, "value_counts", MapType.ofRequired(119, 120,
IntegerType.get(), LongType.get()), "Map of column id to total count, including null and NaN");
Types.NestedField NULL_VALUE_COUNTS = optional(110, "null_value_counts", MapType.ofRequired(121, 122,
IntegerType.get(), LongType.get()), "Map of column id to null value count");
Types.NestedField NAN_VALUE_COUNTS = optional(137, "nan_value_counts", MapType.ofRequired(138, 139,
IntegerType.get(), LongType.get()), "Map of column id to number of NaN values in the column");
Types.NestedField LOWER_BOUNDS = optional(125, "lower_bounds", MapType.ofRequired(126, 127,
IntegerType.get(), BinaryType.get()), "Map of column id to lower bound");
Types.NestedField UPPER_BOUNDS = optional(128, "upper_bounds", MapType.ofRequired(129, 130,
IntegerType.get(), BinaryType.get()), "Map of column id to upper bound");
Types.NestedField KEY_METADATA = optional(131, "key_metadata", BinaryType.get(), "Encryption key metadata blob");
Types.NestedField SPLIT_OFFSETS = optional(132, "split_offsets", ListType.ofRequired(133, LongType.get()),
"Splittable offsets");
Types.NestedField EQUALITY_IDS = optional(135, "equality_ids", ListType.ofRequired(136, IntegerType.get()),
"Equality comparison field IDs");
Types.NestedField SORT_ORDER_ID = optional(140, "sort_order_id", IntegerType.get(), "Sort order ID");
Types.NestedField SPEC_ID = optional(141, "spec_id", IntegerType.get(), "Partition spec ID");
evaluator = new Evaluator(DataFile.getType(EMPTY_STRUCT), fileFilter, caseSensitive);刚开始找到的字段是个unbound,会从第一个字段即元数据信息里面,去寻找可以绑定的字段,第三个参数会决定是不是大小写敏感的。
这里使用的是访问者模式。
绑定字段:
org.apache.iceberg.expressions.NamedReference#bind
@Override
public BoundReference<T> bind(Types.StructType struct, boolean caseSensitive) {
Schema schema = new Schema(struct.fields());
Types.NestedField field = caseSensitive ?
schema.findField(name) :
schema.caseInsensitiveFindField(name);
ValidationException.check(field != null,
"Cannot find field '%s' in struct: %s", name, schema.asStruct());
return new BoundReference<>(field, schema.accessorForField(field.fieldId()));
} public Evaluator(StructType struct, Expression unbound, boolean caseSensitive) {
this.expr = Binder.bind(struct, unbound, caseSensitive);
}
->
public static Expression bind(StructType struct,
Expression expr,
boolean caseSensitive) {
return ExpressionVisitors.visit(expr, new BindVisitor(struct, caseSensitive));
}
->
private BindVisitor(StructType struct, boolean caseSensitive) {
this.struct = struct;
this.caseSensitive = caseSensitive;
}
->
public static <R> R visit(Expression expr, ExpressionVisitor<R> visitor) {
if (expr instanceof Predicate) {
if (expr instanceof BoundPredicate) {
return visitor.predicate((BoundPredicate<?>) expr);
} else {
return visitor.predicate((UnboundPredicate<?>) expr);
}
} else {
switch (expr.op()) {
case TRUE:
return visitor.alwaysTrue();
case FALSE:
return visitor.alwaysFalse();
case NOT:
Not not = (Not) expr;
return visitor.not(visit(not.child(), visitor));
case AND:
And and = (And) expr;
return visitor.and(visit(and.left(), visitor), visit(and.right(), visitor));
case OR:
Or or = (Or) expr;
return visitor.or(visit(or.left(), visitor), visit(or.right(), visitor));
default:
throw new UnsupportedOperationException(
"Unknown operation: " + expr.op());
}
}
}
-->
@Override
public <T> Expression predicate(UnboundPredicate<T> pred) {
return pred.bind(struct, caseSensitive);
}
-->
public Expression bind(StructType struct, boolean caseSensitive) {
BoundTerm<T> bound = term().bind(struct, caseSensitive);
if (literals == null) {
return bindUnaryOperation(bound);
}
if (op() == Operation.IN || op() == Operation.NOT_IN) {
return bindInOperation(bound);
}
return bindLiteralOperation(bound);
}
evaluator的使用主要在以下代码:
return Iterables.transform(
matchingManifests,
manifest -> {
ManifestReader<DataFile> reader = ManifestFiles.read(manifest, io, specsById)
.filterRows(dataFilter)
.filterPartitions(partitionFilter)
.caseSensitive(caseSensitive)
.select(columns);
CloseableIterable<ManifestEntry<DataFile>> entries = reader.entries();
if (ignoreDeleted) {
entries = reader.liveEntries();
}
if (ignoreExisting) {
entries = CloseableIterable.filter(entries,
entry -> entry.status() != ManifestEntry.Status.EXISTING);
}
if (evaluator != null) {
entries = CloseableIterable.filter(entries,
entry -> evaluator.eval((GenericDataFile) entry.file()));
}
entries = CloseableIterable.filter(entries, manifestEntryPredicate);
return entryFn.apply(manifest, entries);
});重点在于evaluator.eval((GenericDataFile) entry.file()));,将数据文件逐个过滤。
public boolean eval(StructLike data) {
return new EvalVisitor().eval(data);
}
private class EvalVisitor extends BoundVisitor<Boolean> {
private StructLike struct;
private boolean eval(StructLike row) {
this.struct = row;
return ExpressionVisitors.visitEvaluator(expr, this); // 重点在这里
}
@Override
public Boolean alwaysTrue() {
return true;
}
………………此处又开始使用访问者模式。
这里我们来看一下iceberg里面有哪些谓语断言(针对BoundPredicate):
@Override
public <T> R predicate(BoundPredicate<T> pred) {
if (pred.isLiteralPredicate()) {
BoundLiteralPredicate<T> literalPred = pred.asLiteralPredicate();
switch (pred.op()) {
case LT:
return lt(pred.term(), literalPred.literal());
case LT_EQ:
return ltEq(pred.term(), literalPred.literal());
case GT:
return gt(pred.term(), literalPred.literal());
case GT_EQ:
return gtEq(pred.term(), literalPred.literal());
case EQ:
return eq(pred.term(), literalPred.literal());
case NOT_EQ:
return notEq(pred.term(), literalPred.literal());
case STARTS_WITH:
return startsWith(pred.term(), literalPred.literal());
case NOT_STARTS_WITH:
return notStartsWith(pred.term(), literalPred.literal());
default:
throw new IllegalStateException("Invalid operation for BoundLiteralPredicate: " + pred.op());
}
} else if (pred.isUnaryPredicate()) {
switch (pred.op()) {
case IS_NULL:
return isNull(pred.term());
case NOT_NULL:
return notNull(pred.term());
case IS_NAN:
return isNaN(pred.term());
case NOT_NAN:
return notNaN(pred.term());
default:
throw new IllegalStateException("Invalid operation for BoundUnaryPredicate: " + pred.op());
}
} else if (pred.isSetPredicate()) {
switch (pred.op()) {
case IN:
return in(pred.term(), pred.asSetPredicate().literalSet());
case NOT_IN:
return notIn(pred.term(), pred.asSetPredicate().literalSet());
default:
throw new IllegalStateException("Invalid operation for BoundSetPredicate: " + pred.op());
}
}
throw new IllegalStateException("Unsupported bound predicate: " + pred.getClass().getName());
}可以看到,这里分为了三个部分:
- isLiteralPredicate:字面量谓语,主要是小于、小于等于、大于、大于等于、等于、不等于、start_with、not_start_with
- isUnaryPredicate:一元运算谓语,主要是is_null not_null is_nan not_nan
- isSetPredicate:集合谓语,主要是in not_in
在进行谓语断言之前,还有几个前置步骤:
- 首先判断是不是谓语断言
- 如果是谓语断言,会判断是不是绑定了字段的,如果没有绑定字段,会进行先进行绑定
- 如果不是谓语断言,则进行关系判定,判断是不是True False Not And Or
未完待续

评论
登录后参与技术讨论,分享你的经验与见解。
登录 / 注册暂无评论