From a0b88165344b50f90cd60359287f8c5f7e5dadbb Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Mon, 2 Nov 2020 13:58:01 -0800 Subject: [PATCH 1/6] Fix timestamp and date transforms. --- .../org/apache/iceberg/transforms/Dates.java | 49 ++++- .../iceberg/transforms/ProjectionUtil.java | 123 ++++++++++++ .../apache/iceberg/transforms/Timestamps.java | 38 +++- .../iceberg/transforms/TransformUtil.java | 3 +- .../TestStringLiteralConversions.java | 53 +++++ .../apache/iceberg/transforms/TestDates.java | 95 +++++++++ .../transforms/TestDatesProjection.java | 190 +++++++++++++++++- .../iceberg/transforms/TestTimestamps.java | 94 +++++++++ .../transforms/TestTimestampsProjection.java | 6 +- 9 files changed, 627 insertions(+), 24 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/transforms/Dates.java b/api/src/main/java/org/apache/iceberg/transforms/Dates.java index 588e757cc3ff..2fae4df4a654 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Dates.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Dates.java @@ -20,7 +20,7 @@ package org.apache.iceberg.transforms; import java.time.Instant; -import java.time.OffsetDateTime; +import java.time.LocalDate; import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; import org.apache.iceberg.expressions.BoundPredicate; @@ -36,7 +36,7 @@ enum Dates implements Transform { MONTH(ChronoUnit.MONTHS, "month"), DAY(ChronoUnit.DAYS, "day"); - private static final OffsetDateTime EPOCH = Instant.ofEpochSecond(0).atOffset(ZoneOffset.UTC); + private static final LocalDate EPOCH = Instant.ofEpochSecond(0).atOffset(ZoneOffset.UTC).toLocalDate(); private final ChronoUnit granularity; private final String name; @@ -55,7 +55,15 @@ public Integer apply(Integer days) { return days; } - return (int) granularity.between(EPOCH, EPOCH.plusDays(days)); + if (days >= 0) { + LocalDate date = EPOCH.plusDays(days); + return (int) granularity.between(EPOCH, date); + } else { + // add 1 day to the value to account for the case where there is exactly 1 unit between the date and epoch + // because the result will always be decremented. + LocalDate date = EPOCH.plusDays(days + 1); + return (int) granularity.between(EPOCH, date) - 1; + } } @Override @@ -99,11 +107,24 @@ public UnboundPredicate project(String fieldName, BoundPredicate projected = ProjectionUtil.truncateInteger(fieldName, pred.asLiteralPredicate(), this); + if (this != DAY) { + return ProjectionUtil.fixInclusiveTimeProjection(projected); + } + + return projected; + } else if (pred.isSetPredicate() && pred.op() == Expression.Operation.IN) { - return ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this); + UnboundPredicate projected = ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this); + if (this != DAY) { + return ProjectionUtil.fixInclusiveTimeProjection(projected); + } + + return projected; } + return null; } @@ -115,11 +136,25 @@ public UnboundPredicate projectStrict(String fieldName, BoundPredicate< if (pred.isUnaryPredicate()) { return Expressions.predicate(pred.op(), fieldName); + } else if (pred.isLiteralPredicate()) { - return ProjectionUtil.truncateIntegerStrict(fieldName, pred.asLiteralPredicate(), this); + UnboundPredicate projected = ProjectionUtil.truncateIntegerStrict( + fieldName, pred.asLiteralPredicate(), this); + if (this != DAY) { + return ProjectionUtil.fixStrictTimeProjection(projected); + } + + return projected; + } else if (pred.isSetPredicate() && pred.op() == Expression.Operation.NOT_IN) { - return ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this); + UnboundPredicate projected = ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this); + if (this != DAY) { + return ProjectionUtil.fixStrictTimeProjection(projected); + } + + return projected; } + return null; } diff --git a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java index 39c6f6ef1f8b..789facc898b7 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java +++ b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java @@ -21,14 +21,17 @@ import java.math.BigDecimal; import java.math.BigInteger; +import java.util.Set; import org.apache.iceberg.expressions.BoundLiteralPredicate; import org.apache.iceberg.expressions.BoundPredicate; import org.apache.iceberg.expressions.BoundSetPredicate; import org.apache.iceberg.expressions.BoundTransform; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.expressions.UnboundPredicate; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; import static org.apache.iceberg.expressions.Expressions.predicate; @@ -254,4 +257,124 @@ static UnboundPredicate transformSet(String fieldName, return predicate(predicate.op(), fieldName, Iterables.transform(predicate.asSetPredicate().literalSet(), transform::apply)); } + + static UnboundPredicate fixInclusiveTimeProjection(UnboundPredicate projected) { + if (projected == null) { + return projected; + } + + // adjust the predicate for values that were 1 larger than the correct transformed value + switch (projected.op()) { + case LT: + if (projected.literal().value() < 0) { + return Expressions.lessThan(projected.term(), projected.literal().value() + 1); + } + + return projected; + + case LT_EQ: + if (projected.literal().value() < 0) { + return Expressions.lessThanOrEqual(projected.term(), projected.literal().value() + 1); + } + + return projected; + + case GT: + case GT_EQ: + // incorrect projected values are already greater than the bound for GT, GT_EQ + return projected; + + case EQ: + if (projected.literal().value() < 0) { + // match either the incorrect value (projectedValue + 1) or the correct value (projectedValue) + return Expressions.in(projected.term(), projected.literal().value(), projected.literal().value() + 1); + } + + return projected; + + case IN: + Set fixedSet = Sets.newHashSet(); + boolean hasNegativeValue = false; + for (Literal lit : projected.literals()) { + Integer value = lit.value(); + fixedSet.add(value); + if (value < 0) { + hasNegativeValue = true; + fixedSet.add(value + 1); + } + } + + if (hasNegativeValue) { + return Expressions.in(projected.term(), fixedSet); + } + + return projected; + + case NOT_IN: + case NOT_EQ: + // there is no inclusive projection for NOT_EQ and NOT_IN + return null; + + default: + return projected; + } + } + + static UnboundPredicate fixStrictTimeProjection(UnboundPredicate projected) { + if (projected == null) { + return null; + } + + switch (projected.op()) { + case LT: + case LT_EQ: + if (projected.literal().value() <= 0) { + // LT and LT_EQ cannot be fixed because adjusting the strict projection may include values that were not + // written incorrectly. for example, if the correct strict projection is x < 5, adjusting to produce x < 6 + // cannot guarantee that all values match the original predicate + return null; + } + + return projected; + + case GT: + case GT_EQ: + // EQ and GT_EQ do not need to be adjusted because the incorrect value is more strict than the projection + // for example, if the correct strict projection is x > 5, the incorrect value, x > 6, is more strict + return projected; + + case EQ: + case IN: + // there is no strict projection for EQ and IN + return null; + + case NOT_EQ: + if (projected.literal().value() < 0) { + return Expressions.notIn(projected.term(), projected.literal().value(), projected.literal().value() + 1); + } + + return projected; + + case NOT_IN: + Set fixedSet = Sets.newHashSet(); + boolean hasNegativeValue = false; + for (Literal lit : projected.literals()) { + Integer value = lit.value(); + fixedSet.add(value); + if (value < 0) { + hasNegativeValue = true; + fixedSet.add(value + 1); + } + } + + if (hasNegativeValue) { + return Expressions.notIn(projected.term(), fixedSet); + } + + return projected; + + default: + return null; + } + } } diff --git a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java index 199fe8e748a2..de4bfcf8cacd 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java @@ -52,12 +52,19 @@ public Integer apply(Long timestampMicros) { return null; } - // discards fractional seconds, not needed for calculation - OffsetDateTime timestamp = Instant - .ofEpochSecond(timestampMicros / 1_000_000) - .atOffset(ZoneOffset.UTC); - - return (int) granularity.between(EPOCH, timestamp); + if (timestampMicros >= 0) { + OffsetDateTime timestamp = Instant + .ofEpochSecond(Math.floorDiv(timestampMicros, 1_000_000), Math.floorMod(timestampMicros, 1_000_000)) + .atOffset(ZoneOffset.UTC); + return (int) granularity.between(EPOCH, timestamp); + } else { + // add 1 micro to the value to account for the case where there is exactly 1 unit between the timestamp and epoch + // because the result will always be decremented. + OffsetDateTime timestamp = Instant + .ofEpochSecond(Math.floorDiv(timestampMicros, 1_000_000), Math.floorMod(timestampMicros + 1, 1_000_000)) + .atOffset(ZoneOffset.UTC); + return (int) granularity.between(EPOCH, timestamp) - 1; + } } @Override @@ -101,11 +108,16 @@ public UnboundPredicate project(String fieldName, BoundPredicate if (pred.isUnaryPredicate()) { return Expressions.predicate(pred.op(), fieldName); + } else if (pred.isLiteralPredicate()) { - return ProjectionUtil.truncateLong(fieldName, pred.asLiteralPredicate(), this); + UnboundPredicate projected = ProjectionUtil.truncateLong(fieldName, pred.asLiteralPredicate(), this); + return ProjectionUtil.fixInclusiveTimeProjection(projected); + } else if (pred.isSetPredicate() && pred.op() == Expression.Operation.IN) { - return ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this); + UnboundPredicate projected = ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this); + return ProjectionUtil.fixInclusiveTimeProjection(projected); } + return null; } @@ -117,11 +129,17 @@ public UnboundPredicate projectStrict(String fieldName, BoundPredicate< if (pred.isUnaryPredicate()) { return Expressions.predicate(pred.op(), fieldName); + } else if (pred.isLiteralPredicate()) { - return ProjectionUtil.truncateLongStrict(fieldName, pred.asLiteralPredicate(), this); + UnboundPredicate projected = ProjectionUtil.truncateLongStrict( + fieldName, pred.asLiteralPredicate(), this); + return ProjectionUtil.fixStrictTimeProjection(projected); + } else if (pred.isSetPredicate() && pred.op() == Expression.Operation.NOT_IN) { - return ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this); + UnboundPredicate projected = ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this); + return ProjectionUtil.fixStrictTimeProjection(projected); } + return null; } diff --git a/api/src/main/java/org/apache/iceberg/transforms/TransformUtil.java b/api/src/main/java/org/apache/iceberg/transforms/TransformUtil.java index b2811389d876..5e9cd5aa5403 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/TransformUtil.java +++ b/api/src/main/java/org/apache/iceberg/transforms/TransformUtil.java @@ -41,7 +41,8 @@ static String humanYear(int yearOrdinal) { } static String humanMonth(int monthOrdinal) { - return String.format("%04d-%02d", EPOCH_YEAR + (monthOrdinal / 12), 1 + (monthOrdinal % 12)); + return String.format("%04d-%02d", + EPOCH_YEAR + Math.floorDiv(monthOrdinal, 12), 1 + Math.floorMod(monthOrdinal, 12)); } static String humanDay(int dayOrdinal) { diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestStringLiteralConversions.java b/api/src/test/java/org/apache/iceberg/expressions/TestStringLiteralConversions.java index 329b1433b8ed..d2812a731817 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestStringLiteralConversions.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestStringLiteralConversions.java @@ -57,6 +57,22 @@ public void testStringToDateLiteral() { Assert.assertEquals("Date should match", avroValue, (int) date.value()); } + @Test + public void testNegativeStringToDateLiteral() { + Literal dateStr = Literal.of("1969-12-30"); + Literal date = dateStr.to(Types.DateType.get()); + + // use Avro's date conversion to validate the result + Schema avroSchema = LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT)); + TimeConversions.DateConversion avroConversion = new TimeConversions.DateConversion(); + int avroValue = avroConversion.toInt( + LocalDate.of(1969, 12, 30), + avroSchema, avroSchema.getLogicalType()); + + Assert.assertEquals("Date should be -2", -2, (int) date.value()); + Assert.assertEquals("Date should match", avroValue, (int) date.value()); + } + @Test public void testStringToTimeLiteral() { // use Avro's time conversion to validate the result @@ -106,6 +122,43 @@ public void testStringToTimestampLiteral() { avroValue, (long) timestamp.value()); } + @Test + public void testNegativeStringToTimestampLiteral() { + // use Avro's timestamp conversion to validate the result + Schema avroSchema = LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG)); + TimeConversions.TimestampMicrosConversion avroConversion = + new TimeConversions.TimestampMicrosConversion(); + + // Timestamp with explicit UTC offset, +00:00 + Literal timestampStr = Literal.of("1969-12-31T23:59:58.999999+00:00"); + Literal timestamp = timestampStr.to(Types.TimestampType.withZone()); + long avroValue = avroConversion.toLong( + LocalDateTime.of(1969, 12, 31, 23, 59, 58, 999999 * 1_000).toInstant(ZoneOffset.UTC), + avroSchema, avroSchema.getLogicalType()); + + Assert.assertEquals("Timestamp should match", avroValue, (long) timestamp.value()); + Assert.assertEquals("Timestamp should be -1_000_001", -1_000_001, (long) timestamp.value()); + + // Timestamp without an explicit zone should be UTC (equal to the previous converted value) + timestampStr = Literal.of("1969-12-31T23:59:58.999999"); + timestamp = timestampStr.to(Types.TimestampType.withoutZone()); + + Assert.assertEquals("Timestamp without zone should match UTC", + avroValue, (long) timestamp.value()); + + // Timestamp with an explicit offset should be adjusted to UTC + timestampStr = Literal.of("1969-12-31T16:59:58.999999-07:00"); + timestamp = timestampStr.to(Types.TimestampType.withZone()); + avroValue = avroConversion.toLong( + LocalDateTime.of(1969, 12, 31, 23, 59, 58, 999999 * 1_000).toInstant(ZoneOffset.UTC), + avroSchema, avroSchema.getLogicalType()); + + Assert.assertEquals("Timestamp without zone should match UTC", + avroValue, (long) timestamp.value()); + Assert.assertEquals("Timestamp without zone should be -1_000_001", -1_000_001, (long) timestamp.value()); + + } + @Test(expected = DateTimeException.class) public void testTimestampWithZoneWithoutZoneInLiteral() { // Zone must be present in literals when converting to timestamp with zone diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDates.java b/api/src/test/java/org/apache/iceberg/transforms/TestDates.java index c344a975f2aa..6414b74e73bc 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestDates.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestDates.java @@ -26,6 +26,29 @@ import org.junit.Test; public class TestDates { + @Test + public void testDateTransform() { + Types.DateType type = Types.DateType.get(); + Literal d = Literal.of("2017-12-01").to(type); + Literal pd = Literal.of("1970-01-01").to(type); + Literal nd = Literal.of("1969-12-31").to(type); + + Transform years = Transforms.year(type); + Assert.assertEquals("Should produce 2017 - 1970 = 47", 47, (int) years.apply(d.value())); + Assert.assertEquals("Should produce 1970 - 1970 = 0", 0, (int) years.apply(pd.value())); + Assert.assertEquals("Should produce 1969 - 1970 = -1", -1, (int) years.apply(nd.value())); + + Transform months = Transforms.month(type); + Assert.assertEquals("Should produce 47 * 12 + 11 = 575", 575, (int) months.apply(d.value())); + Assert.assertEquals("Should produce 0 * 12 + 0 = 0", 0, (int) months.apply(pd.value())); + Assert.assertEquals("Should produce -1", -1, (int) months.apply(nd.value())); + + Transform days = Transforms.day(type); + Assert.assertEquals("Should produce 17501", 17501, (int) days.apply(d.value())); + Assert.assertEquals("Should produce 0 * 365 + 0 = 0", 0, (int) days.apply(pd.value())); + Assert.assertEquals("Should produce -1", -1, (int) days.apply(nd.value())); + } + @Test public void testDateToHumanString() { Types.DateType type = Types.DateType.get(); @@ -44,6 +67,78 @@ public void testDateToHumanString() { "2017-12-01", day.toHumanString(day.apply(date.value()))); } + @Test + public void testNegativeDateToHumanString() { + Types.DateType type = Types.DateType.get(); + Literal date = Literal.of("1969-12-30").to(type); + + Transform year = Transforms.year(type); + Assert.assertEquals("Should produce the correct Human string", + "1969", year.toHumanString(year.apply(date.value()))); + + Transform month = Transforms.month(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-12", month.toHumanString(month.apply(date.value()))); + + Transform day = Transforms.day(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-12-30", day.toHumanString(day.apply(date.value()))); + } + + @Test + public void testDateToHumanStringLowerBound() { + Types.DateType type = Types.DateType.get(); + Literal date = Literal.of("1970-01-01").to(type); + + Transform year = Transforms.year(type); + Assert.assertEquals("Should produce the correct Human string", + "1970", year.toHumanString(year.apply(date.value()))); + + Transform month = Transforms.month(type); + Assert.assertEquals("Should produce the correct Human string", + "1970-01", month.toHumanString(month.apply(date.value()))); + + Transform day = Transforms.day(type); + Assert.assertEquals("Should produce the correct Human string", + "1970-01-01", day.toHumanString(day.apply(date.value()))); + } + + @Test + public void testNegativeDateToHumanStringLowerBound() { + Types.DateType type = Types.DateType.get(); + Literal date = Literal.of("1969-01-01").to(type); + + Transform year = Transforms.year(type); + Assert.assertEquals("Should produce the correct Human string", + "1969", year.toHumanString(year.apply(date.value()))); + + Transform month = Transforms.month(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-01", month.toHumanString(month.apply(date.value()))); + + Transform day = Transforms.day(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-01-01", day.toHumanString(day.apply(date.value()))); + } + + @Test + public void testNegativeDateToHumanStringUpperBound() { + Types.DateType type = Types.DateType.get(); + Literal date = Literal.of("1969-12-31").to(type); + + Transform year = Transforms.year(type); + Assert.assertEquals("Should produce the correct Human string", + "1969", year.toHumanString(year.apply(date.value()))); + + Transform month = Transforms.month(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-12", month.toHumanString(month.apply(date.value()))); + + Transform day = Transforms.day(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-12-31", day.toHumanString(day.apply(date.value()))); + } + @Test public void testNullHumanString() { Types.DateType type = Types.DateType.get(); diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java index 475ec7f1afcd..ede55dc08ad5 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java @@ -74,14 +74,14 @@ public void assertProjectionStrictValue(PartitionSpec spec, UnboundPredicate Expression.Operation expectedOp) { Expression projection = Projections.strict(spec).project(filter); - Assert.assertEquals(projection.op(), expectedOp); + Assert.assertEquals(expectedOp, projection.op()); } public void assertProjectionInclusiveValue(PartitionSpec spec, UnboundPredicate filter, Expression.Operation expectedOp) { Expression projection = Projections.inclusive(spec).project(filter); - Assert.assertEquals(projection.op(), expectedOp); + Assert.assertEquals(expectedOp, projection.op()); } public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate filter, @@ -89,7 +89,7 @@ public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate fi Expression projection = Projections.inclusive(spec).project(filter); UnboundPredicate predicate = assertAndUnwrapUnbound(projection); - Assert.assertEquals(predicate.op(), expectedOp); + Assert.assertEquals(expectedOp, predicate.op()); Assert.assertNotEquals("Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op()); @@ -124,6 +124,25 @@ public void testMonthStrictLowerBound() { assertProjectionStrictValue(spec, in("date", anotherDate, date), Expression.Operation.FALSE); } + @Test + public void testNegativeMonthStrictLowerBound() { + Integer date = (Integer) Literal.of("1970-01-01").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); + + // the boundary cannot be projected because fixing strict projection must fix cases where value + 1 = 0 + assertProjectionStrictValue(spec, lessThan("date", date), Expression.Operation.FALSE); + assertProjectionStrictValue(spec, lessThanOrEqual("date", date), Expression.Operation.FALSE); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1970-01"); + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1969-12"); + assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "1970-01"); + assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); + + Integer anotherDate = (Integer) Literal.of("1969-12-31").to(TYPE).value(); + assertProjectionStrict(spec, notIn("date", date, anotherDate), + Expression.Operation.NOT_IN, "[1969-12, 1970-01]"); + assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE); + } + @Test public void testMonthStrictUpperBound() { Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); @@ -142,6 +161,24 @@ public void testMonthStrictUpperBound() { assertProjectionStrictValue(spec, in("date", anotherDate, date), Expression.Operation.FALSE); } + @Test + public void testNegativeMonthStrictUpperBound() { + Integer date = (Integer) Literal.of("1969-12-31").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); + + assertProjectionStrictValue(spec, lessThan("date", date), Expression.Operation.FALSE); + assertProjectionStrictValue(spec, lessThanOrEqual("date", date), Expression.Operation.FALSE); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1969-12"); + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1969-12"); + assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_IN, "[1969-12, 1970-01]"); + assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); + + Integer anotherDate = (Integer) Literal.of("1970-01-01").to(TYPE).value(); + assertProjectionStrict(spec, notIn("date", date, anotherDate), + Expression.Operation.NOT_IN, "[1969-12, 1970-01]"); + assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE); + } + @Test public void testMonthInclusiveLowerBound() { Integer date = (Integer) Literal.of("2017-12-01").to(TYPE).value(); @@ -160,6 +197,24 @@ public void testMonthInclusiveLowerBound() { assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE); } + @Test + public void testNegativeMonthInclusiveLowerBound() { + Integer date = (Integer) Literal.of("1970-01-01").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); + + assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "1970-01"); + assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "1970-01"); + assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "1970-01"); + assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "1970-01"); + assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "1970-01"); + assertProjectionInclusiveValue(spec, notEqual("date", date), Expression.Operation.TRUE); + + Integer anotherDate = (Integer) Literal.of("1969-12-31").to(TYPE).value(); + assertProjectionInclusive(spec, in("date", date, anotherDate), + Expression.Operation.IN, "[1969-12, 1970-01]"); + assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE); + } + @Test public void testMonthInclusiveUpperBound() { Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); @@ -178,6 +233,24 @@ public void testMonthInclusiveUpperBound() { assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE); } + @Test + public void testNegativeMonthInclusiveUpperBound() { + Integer date = (Integer) Literal.of("1969-12-31").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); + + assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "1970-01"); + assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "1970-01"); + assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "1970-01"); + assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "1969-12"); + assertProjectionInclusive(spec, equal("date", date), Expression.Operation.IN, "[1969-12, 1970-01]"); + assertProjectionInclusiveValue(spec, notEqual("date", date), Expression.Operation.TRUE); + + Integer anotherDate = (Integer) Literal.of("1969-01-01").to(TYPE).value(); + assertProjectionInclusive(spec, in("date", date, anotherDate), + Expression.Operation.IN, "[1969-01, 1969-02, 1969-12, 1970-01]"); + assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE); + } + @Test public void testDayStrict() { Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value(); @@ -198,6 +271,26 @@ public void testDayStrict() { assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE); } + @Test + public void testNegativeDayStrict() { + Integer date = (Integer) Literal.of("1969-12-30").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("date").build(); + + assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "1969-12-30"); + // should be the same date for <= + assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "1969-12-31"); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1969-12-30"); + // should be the same date for >= + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1969-12-29"); + assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "1969-12-30"); + assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); + + Integer anotherDate = (Integer) Literal.of("1969-12-28").to(TYPE).value(); + assertProjectionStrict(spec, notIn("date", date, anotherDate), + Expression.Operation.NOT_IN, "[1969-12-28, 1969-12-30]"); + assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE); + } + @Test public void testDayInclusive() { Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value(); @@ -216,6 +309,24 @@ public void testDayInclusive() { assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE); } + @Test + public void testNegativeDayInclusive() { + Integer date = (Integer) Literal.of("1969-12-30").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("date").build(); + + assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "1969-12-29"); + assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "1969-12-30"); + assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "1969-12-31"); + assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "1969-12-30"); + assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "1969-12-30"); + assertProjectionInclusiveValue(spec, notEqual("date", date), Expression.Operation.TRUE); + + Integer anotherDate = (Integer) Literal.of("1969-12-28").to(TYPE).value(); + assertProjectionInclusive(spec, in("date", date, anotherDate), + Expression.Operation.IN, "[1969-12-28, 1969-12-30]"); + assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE); + } + @Test public void testYearStrictLowerBound() { Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value(); @@ -234,6 +345,25 @@ public void testYearStrictLowerBound() { assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE); } + @Test + public void testNegativeYearStrictLowerBound() { + Integer date = (Integer) Literal.of("1970-01-01").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build(); + + // the boundary cannot be projected because fixing strict projection must fix cases where value + 1 = 0 + assertProjectionStrictValue(spec, lessThan("date", date), Expression.Operation.FALSE); + assertProjectionStrictValue(spec, lessThanOrEqual("date", date), Expression.Operation.FALSE); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1970"); + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1969"); + assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "1970"); + assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); + + Integer anotherDate = (Integer) Literal.of("1969-12-31").to(TYPE).value(); + assertProjectionStrict(spec, notIn("date", date, anotherDate), + Expression.Operation.NOT_IN, "[1969, 1970]"); + assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE); + } + @Test public void testYearStrictUpperBound() { Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); @@ -252,6 +382,24 @@ public void testYearStrictUpperBound() { assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE); } + @Test + public void testNegativeYearStrictUpperBound() { + Integer date = (Integer) Literal.of("1969-12-31").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build(); + + assertProjectionStrictValue(spec, lessThan("date", date), Expression.Operation.FALSE); + assertProjectionStrictValue(spec, lessThanOrEqual("date", date), Expression.Operation.FALSE); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1969"); + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1969"); + assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_IN, "[1969, 1970]"); + assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); + + Integer anotherDate = (Integer) Literal.of("1970-01-01").to(TYPE).value(); + assertProjectionStrict(spec, notIn("date", date, anotherDate), + Expression.Operation.NOT_IN, "[1969, 1970]"); + assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE); + } + @Test public void testYearInclusiveLowerBound() { Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value(); @@ -270,6 +418,24 @@ public void testYearInclusiveLowerBound() { assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE); } + @Test + public void testNegativeYearInclusiveLowerBound() { + Integer date = (Integer) Literal.of("1970-01-01").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build(); + + assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "1970"); + assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "1970"); + assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "1970"); + assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "1970"); + assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "1970"); + assertProjectionInclusiveValue(spec, notEqual("date", date), Expression.Operation.TRUE); + + Integer anotherDate = (Integer) Literal.of("1969-12-31").to(TYPE).value(); + assertProjectionInclusive(spec, in("date", date, anotherDate), + Expression.Operation.IN, "[1969, 1970]"); + assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE); + } + @Test public void testYearInclusiveUpperBound() { Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); @@ -287,4 +453,22 @@ public void testYearInclusiveUpperBound() { Expression.Operation.IN, "[2016, 2017]"); assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE); } + + @Test + public void testNegativeYearInclusiveUpperBound() { + Integer date = (Integer) Literal.of("1969-12-31").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build(); + + assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "1970"); + assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "1970"); + assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "1970"); + assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "1969"); + assertProjectionInclusive(spec, equal("date", date), Expression.Operation.IN, "[1969, 1970]"); + assertProjectionInclusiveValue(spec, notEqual("date", date), Expression.Operation.TRUE); + + Integer anotherDate = (Integer) Literal.of("1969-01-01").to(TYPE).value(); + assertProjectionInclusive(spec, in("date", date, anotherDate), + Expression.Operation.IN, "[1969, 1970]"); + assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE); + } } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java index ed71a8b30400..8a4ba8a40e37 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java @@ -26,6 +26,34 @@ import org.junit.Test; public class TestTimestamps { + @Test + public void testTimestampTransform() { + Types.TimestampType type = Types.TimestampType.withoutZone(); + Literal ts = Literal.of("2017-12-01T10:12:55.038194").to(type); + Literal pts = Literal.of("1970-01-01T00:00:01.000001").to(type); + Literal nts = Literal.of("1969-12-31T23:59:58.999999").to(type); + + Transform years = Transforms.year(type); + Assert.assertEquals("Should produce 2017 - 1970 = 47", 47, (int) years.apply(ts.value())); + Assert.assertEquals("Should produce 1970 - 1970 = 0", 0, (int) years.apply(pts.value())); + Assert.assertEquals("Should produce 1969 - 1970 = -1", -1, (int) years.apply(nts.value())); + + Transform months = Transforms.month(type); + Assert.assertEquals("Should produce 47 * 12 + 11 = 575", 575, (int) months.apply(ts.value())); + Assert.assertEquals("Should produce 0 * 12 + 0 = 0", 0, (int) months.apply(pts.value())); + Assert.assertEquals("Should produce -1", -1, (int) months.apply(nts.value())); + + Transform days = Transforms.day(type); + Assert.assertEquals("Should produce 17501", 17501, (int) days.apply(ts.value())); + Assert.assertEquals("Should produce 0 * 365 + 0 = 0", 0, (int) days.apply(pts.value())); + Assert.assertEquals("Should produce -1", -1, (int) days.apply(nts.value())); + + Transform hours = Transforms.hour(type); + Assert.assertEquals("Should produce 17501 * 24 + 10", 420034, (int) hours.apply(ts.value())); + Assert.assertEquals("Should produce 0 * 24 + 0 = 0", 0, (int) hours.apply(pts.value())); + Assert.assertEquals("Should produce -1", -1, (int) hours.apply(nts.value())); + } + @Test public void testTimestampWithoutZoneToHumanString() { Types.TimestampType type = Types.TimestampType.withoutZone(); @@ -48,6 +76,72 @@ public void testTimestampWithoutZoneToHumanString() { "2017-12-01-10", hour.toHumanString(hour.apply(date.value()))); } + @Test + public void testNegativeTimestampWithoutZoneToHumanString() { + Types.TimestampType type = Types.TimestampType.withoutZone(); + Literal date = Literal.of("1969-12-30T10:12:55.038194").to(type); + + Transform year = Transforms.year(type); + Assert.assertEquals("Should produce the correct Human string", + "1969", year.toHumanString(year.apply(date.value()))); + + Transform month = Transforms.month(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-12", month.toHumanString(month.apply(date.value()))); + + Transform day = Transforms.day(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-12-30", day.toHumanString(day.apply(date.value()))); + + Transform hour = Transforms.hour(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-12-30-10", hour.toHumanString(hour.apply(date.value()))); + } + + @Test + public void testNegativeTimestampWithoutZoneToHumanStringLowerBound() { + Types.TimestampType type = Types.TimestampType.withoutZone(); + Literal date = Literal.of("1969-12-30T00:00:00.000000").to(type); + + Transform year = Transforms.year(type); + Assert.assertEquals("Should produce the correct Human string", + "1969", year.toHumanString(year.apply(date.value()))); + + Transform month = Transforms.month(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-12", month.toHumanString(month.apply(date.value()))); + + Transform day = Transforms.day(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-12-30", day.toHumanString(day.apply(date.value()))); + + Transform hour = Transforms.hour(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-12-30-00", hour.toHumanString(hour.apply(date.value()))); + } + + @Test + public void testNegativeTimestampWithoutZoneToHumanStringUpperBound() { + Types.TimestampType type = Types.TimestampType.withoutZone(); + Literal date = Literal.of("1969-12-31T23:59:59.999999").to(type); + + Transform year = Transforms.year(type); + Assert.assertEquals("Should produce the correct Human string", + "1969", year.toHumanString(year.apply(date.value()))); + + Transform month = Transforms.month(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-12", month.toHumanString(month.apply(date.value()))); + + Transform day = Transforms.day(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-12-31", day.toHumanString(day.apply(date.value()))); + + Transform hour = Transforms.hour(type); + Assert.assertEquals("Should produce the correct Human string", + "1969-12-31-23", hour.toHumanString(hour.apply(date.value()))); + } + @Test public void testTimestampWithZoneToHumanString() { Types.TimestampType type = Types.TimestampType.withZone(); diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java index 3af41473d467..8526a91071ff 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java @@ -74,14 +74,14 @@ public void assertProjectionStrictValue(PartitionSpec spec, UnboundPredicate Expression.Operation expectedOp) { Expression projection = Projections.strict(spec).project(filter); - Assert.assertEquals(projection.op(), expectedOp); + Assert.assertEquals(expectedOp, projection.op()); } public void assertProjectionInclusiveValue(PartitionSpec spec, UnboundPredicate filter, Expression.Operation expectedOp) { Expression projection = Projections.inclusive(spec).project(filter); - Assert.assertEquals(projection.op(), expectedOp); + Assert.assertEquals(expectedOp, projection.op()); } public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate filter, @@ -89,7 +89,7 @@ public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate fi Expression projection = Projections.inclusive(spec).project(filter); UnboundPredicate predicate = assertAndUnwrapUnbound(projection); - Assert.assertEquals(predicate.op(), expectedOp); + Assert.assertEquals(expectedOp, predicate.op()); Assert.assertNotEquals("Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op()); From caad9b5564e9f342a35273f56ab7cf1fce85bc6d Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Tue, 29 Dec 2020 10:46:34 -0800 Subject: [PATCH 2/6] Fix checkstyle. --- .../java/org/apache/iceberg/transforms/TestDates.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDates.java b/api/src/test/java/org/apache/iceberg/transforms/TestDates.java index 6414b74e73bc..f9479f8c3dae 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestDates.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestDates.java @@ -29,22 +29,22 @@ public class TestDates { @Test public void testDateTransform() { Types.DateType type = Types.DateType.get(); - Literal d = Literal.of("2017-12-01").to(type); + Literal date = Literal.of("2017-12-01").to(type); Literal pd = Literal.of("1970-01-01").to(type); Literal nd = Literal.of("1969-12-31").to(type); Transform years = Transforms.year(type); - Assert.assertEquals("Should produce 2017 - 1970 = 47", 47, (int) years.apply(d.value())); + Assert.assertEquals("Should produce 2017 - 1970 = 47", 47, (int) years.apply(date.value())); Assert.assertEquals("Should produce 1970 - 1970 = 0", 0, (int) years.apply(pd.value())); Assert.assertEquals("Should produce 1969 - 1970 = -1", -1, (int) years.apply(nd.value())); Transform months = Transforms.month(type); - Assert.assertEquals("Should produce 47 * 12 + 11 = 575", 575, (int) months.apply(d.value())); + Assert.assertEquals("Should produce 47 * 12 + 11 = 575", 575, (int) months.apply(date.value())); Assert.assertEquals("Should produce 0 * 12 + 0 = 0", 0, (int) months.apply(pd.value())); Assert.assertEquals("Should produce -1", -1, (int) months.apply(nd.value())); Transform days = Transforms.day(type); - Assert.assertEquals("Should produce 17501", 17501, (int) days.apply(d.value())); + Assert.assertEquals("Should produce 17501", 17501, (int) days.apply(date.value())); Assert.assertEquals("Should produce 0 * 365 + 0 = 0", 0, (int) days.apply(pd.value())); Assert.assertEquals("Should produce -1", -1, (int) days.apply(nd.value())); } From b4eb6e6dff84c35e02884e73d8de6bc34c9e6565 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Tue, 29 Dec 2020 11:37:57 -0800 Subject: [PATCH 3/6] Add more tests. --- .../apache/iceberg/transforms/Timestamps.java | 8 +- .../transforms/TestDatesProjection.java | 66 +++++-- .../transforms/TestTimestampsProjection.java | 181 ++++++++++++++++++ 3 files changed, 238 insertions(+), 17 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java index de4bfcf8cacd..75735f77158c 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java @@ -54,14 +54,18 @@ public Integer apply(Long timestampMicros) { if (timestampMicros >= 0) { OffsetDateTime timestamp = Instant - .ofEpochSecond(Math.floorDiv(timestampMicros, 1_000_000), Math.floorMod(timestampMicros, 1_000_000)) + .ofEpochSecond( + Math.floorDiv(timestampMicros, 1_000_000), + Math.floorMod(timestampMicros, 1_000_000) * 1000) .atOffset(ZoneOffset.UTC); return (int) granularity.between(EPOCH, timestamp); } else { // add 1 micro to the value to account for the case where there is exactly 1 unit between the timestamp and epoch // because the result will always be decremented. OffsetDateTime timestamp = Instant - .ofEpochSecond(Math.floorDiv(timestampMicros, 1_000_000), Math.floorMod(timestampMicros + 1, 1_000_000)) + .ofEpochSecond( + Math.floorDiv(timestampMicros, 1_000_000), + Math.floorMod(timestampMicros + 1, 1_000_000) * 1000) .atOffset(ZoneOffset.UTC); return (int) granularity.between(EPOCH, timestamp) - 1; } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java index ede55dc08ad5..15830d6583e2 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java @@ -106,6 +106,43 @@ public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate fi } } + @Test + public void testMonthStrictEpoch() { + Integer date = (Integer) Literal.of("1970-01-01").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); + + // the boundary cannot be projected because fixing strict projection must fix cases where value + 1 = 0 + assertProjectionStrictValue(spec, lessThan("date", date), Expression.Operation.FALSE); + assertProjectionStrictValue(spec, lessThanOrEqual("date", date), Expression.Operation.FALSE); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1970-01"); + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1969-12"); + assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "1970-01"); + assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); + + Integer anotherDate = (Integer) Literal.of("1969-12-31").to(TYPE).value(); + assertProjectionStrict(spec, notIn("date", date, anotherDate), + Expression.Operation.NOT_IN, "[1969-12, 1970-01]"); + assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE); + } + + @Test + public void testMonthInclusiveEpoch() { + Integer date = (Integer) Literal.of("1970-01-01").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); + + assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "1970-01"); + assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "1970-01"); + assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "1970-01"); + assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "1970-01"); + assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "1970-01"); + assertProjectionInclusiveValue(spec, notEqual("date", date), Expression.Operation.TRUE); + + Integer anotherDate = (Integer) Literal.of("1969-12-31").to(TYPE).value(); + assertProjectionInclusive(spec, in("date", date, anotherDate), + Expression.Operation.IN, "[1969-12, 1970-01]"); + assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE); + } + @Test public void testMonthStrictLowerBound() { Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value(); @@ -126,20 +163,19 @@ public void testMonthStrictLowerBound() { @Test public void testNegativeMonthStrictLowerBound() { - Integer date = (Integer) Literal.of("1970-01-01").to(TYPE).value(); + Integer date = (Integer) Literal.of("1969-01-01").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); - // the boundary cannot be projected because fixing strict projection must fix cases where value + 1 = 0 assertProjectionStrictValue(spec, lessThan("date", date), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThanOrEqual("date", date), Expression.Operation.FALSE); - assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1970-01"); - assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1969-12"); - assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "1970-01"); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1969-01"); + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1968-12"); + assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_IN, "[1969-01, 1969-02]"); assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); Integer anotherDate = (Integer) Literal.of("1969-12-31").to(TYPE).value(); assertProjectionStrict(spec, notIn("date", date, anotherDate), - Expression.Operation.NOT_IN, "[1969-12, 1970-01]"); + Expression.Operation.NOT_IN, "[1969-01, 1969-02, 1969-12, 1970-01]"); assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE); } @@ -173,9 +209,9 @@ public void testNegativeMonthStrictUpperBound() { assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_IN, "[1969-12, 1970-01]"); assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); - Integer anotherDate = (Integer) Literal.of("1970-01-01").to(TYPE).value(); + Integer anotherDate = (Integer) Literal.of("1969-11-01").to(TYPE).value(); assertProjectionStrict(spec, notIn("date", date, anotherDate), - Expression.Operation.NOT_IN, "[1969-12, 1970-01]"); + Expression.Operation.NOT_IN, "[1969-11, 1969-12, 1970-01]"); assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE); } @@ -199,19 +235,19 @@ public void testMonthInclusiveLowerBound() { @Test public void testNegativeMonthInclusiveLowerBound() { - Integer date = (Integer) Literal.of("1970-01-01").to(TYPE).value(); + Integer date = (Integer) Literal.of("1969-01-01").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); - assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "1970-01"); - assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "1970-01"); - assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "1970-01"); - assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "1970-01"); - assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "1970-01"); + assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "1969-01"); + assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "1969-02"); + assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "1969-01"); + assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "1969-01"); + assertProjectionInclusive(spec, equal("date", date), Expression.Operation.IN, "[1969-01, 1969-02]"); assertProjectionInclusiveValue(spec, notEqual("date", date), Expression.Operation.TRUE); Integer anotherDate = (Integer) Literal.of("1969-12-31").to(TYPE).value(); assertProjectionInclusive(spec, in("date", date, anotherDate), - Expression.Operation.IN, "[1969-12, 1970-01]"); + Expression.Operation.IN, "[1969-01, 1969-02, 1969-12, 1970-01]"); assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE); } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java index 8526a91071ff..0237219566b7 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java @@ -106,6 +106,43 @@ public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate fi } } + @Test + public void testDayStrictEpoch() { + Long date = (long) Literal.of("1970-01-01T00:00:00.00000").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); + + // the boundary cannot be projected because fixing strict projection must fix cases where value + 1 = 0 + assertProjectionStrictValue(spec, lessThan("timestamp", date), Expression.Operation.FALSE); + assertProjectionStrictValue(spec, lessThanOrEqual("timestamp", date), Expression.Operation.FALSE); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1970-01-01"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1969-12-31"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "1970-01-01"); + assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE); + + Long anotherDate = (long) Literal.of("1970-01-02T00:00:00.00000").to(TYPE).value(); + assertProjectionStrict(spec, notIn("timestamp", date, anotherDate), + Expression.Operation.NOT_IN, "[1970-01-01, 1970-01-02]"); + assertProjectionStrictValue(spec, in("timestamp", date, anotherDate), Expression.Operation.FALSE); + } + + @Test + public void testDayInclusiveEpoch() { + Long date = (long) Literal.of("1970-01-01T00:00:00.00000").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); + + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "1970-01-01"); + assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "1970-01-01"); + assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "1970-01-01"); + assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "1970-01-01"); + assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "1970-01-01"); + assertProjectionInclusiveValue(spec, notEqual("timestamp", date), Expression.Operation.TRUE); + + Long anotherDate = (long) Literal.of("1970-01-02T00:00:00.00000").to(TYPE).value(); + assertProjectionInclusive(spec, in("timestamp", date, anotherDate), + Expression.Operation.IN, "[1970-01-01, 1970-01-02]"); + assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE); + } + @Test public void testMonthStrictLowerBound() { Long date = (long) Literal.of("2017-12-01T00:00:00.00000").to(TYPE).value(); @@ -124,6 +161,24 @@ public void testMonthStrictLowerBound() { assertProjectionStrictValue(spec, in("timestamp", anotherDate, date), Expression.Operation.FALSE); } + @Test + public void testNegativeMonthStrictLowerBound() { + Long date = (long) Literal.of("1969-01-01T00:00:00.00000").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); + + assertProjectionStrictValue(spec, lessThan("timestamp", date), Expression.Operation.FALSE); + assertProjectionStrictValue(spec, lessThanOrEqual("timestamp", date), Expression.Operation.FALSE); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1969-01"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1968-12"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_IN, "[1969-01, 1969-02]"); + assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE); + + Long anotherDate = (long) Literal.of("1969-03-01T00:00:00.00000").to(TYPE).value(); + assertProjectionStrict(spec, notIn("timestamp", anotherDate, date), + Expression.Operation.NOT_IN, "[1969-01, 1969-02, 1969-03, 1969-04]"); + assertProjectionStrictValue(spec, in("timestamp", anotherDate, date), Expression.Operation.FALSE); + } + @Test public void testMonthStrictUpperBound() { Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value(); @@ -142,6 +197,24 @@ public void testMonthStrictUpperBound() { assertProjectionStrictValue(spec, in("timestamp", anotherDate, date), Expression.Operation.FALSE); } + @Test + public void testNegativeMonthStrictUpperBound() { + Long date = (long) Literal.of("1969-12-31T23:59:59.999999").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); + + assertProjectionStrictValue(spec, lessThan("timestamp", date), Expression.Operation.FALSE); + assertProjectionStrictValue(spec, lessThanOrEqual("timestamp", date), Expression.Operation.FALSE); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1969-12"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1969-12"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_IN, "[1969-12, 1970-01]"); + assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE); + + Long anotherDate = (long) Literal.of("1970-02-01T00:00:00.00000").to(TYPE).value(); + assertProjectionStrict(spec, notIn("timestamp", anotherDate, date), + Expression.Operation.NOT_IN, "[1969-12, 1970-01, 1970-02]"); + assertProjectionStrictValue(spec, in("timestamp", anotherDate, date), Expression.Operation.FALSE); + } + @Test public void testMonthInclusiveLowerBound() { Long date = (long) Literal.of("2017-12-01T00:00:00.00000").to(TYPE).value(); @@ -160,6 +233,24 @@ public void testMonthInclusiveLowerBound() { assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE); } + @Test + public void testNegativeMonthInclusiveLowerBound() { + Long date = (long) Literal.of("1969-01-01T00:00:00.00000").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); + + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "1969-01"); + assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "1969-02"); + assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "1969-01"); + assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "1969-01"); + assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.IN, "[1969-01, 1969-02]"); + assertProjectionInclusiveValue(spec, notEqual("timestamp", date), Expression.Operation.TRUE); + + Long anotherDate = (long) Literal.of("1969-03-01T00:00:00.00000").to(TYPE).value(); + assertProjectionInclusive(spec, in("timestamp", date, anotherDate), + Expression.Operation.IN, "[1969-01, 1969-02, 1969-03, 1969-04]"); + assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE); + } + @Test public void testMonthInclusiveUpperBound() { Long date = (long) Literal.of("2017-12-01T23:59:59.999999").to(TYPE).value(); @@ -178,6 +269,24 @@ public void testMonthInclusiveUpperBound() { assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE); } + @Test + public void testNegativeMonthInclusiveUpperBound() { + Long date = (long) Literal.of("1969-12-31T23:59:59.999999").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); + + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "1970-01"); + assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "1970-01"); + assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "1970-01"); + assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "1969-12"); + assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.IN, "[1969-12, 1970-01]"); + assertProjectionInclusiveValue(spec, notEqual("timestamp", date), Expression.Operation.TRUE); + + Long anotherDate = (long) Literal.of("1970-01-01T00:00:00.00000").to(TYPE).value(); + assertProjectionInclusive(spec, in("timestamp", date, anotherDate), + Expression.Operation.IN, "[1969-12, 1970-01]"); + assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE); + } + @Test public void testDayStrictLowerBound() { Long date = (long) Literal.of("2017-12-01T00:00:00.00000").to(TYPE).value(); @@ -196,6 +305,24 @@ public void testDayStrictLowerBound() { assertProjectionStrictValue(spec, in("timestamp", date, anotherDate), Expression.Operation.FALSE); } + @Test + public void testNegativeDayStrictLowerBound() { + Long date = (long) Literal.of("1969-01-01T00:00:00.00000").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); + + assertProjectionStrictValue(spec, lessThan("timestamp", date), Expression.Operation.FALSE); + assertProjectionStrictValue(spec, lessThanOrEqual("timestamp", date), Expression.Operation.FALSE); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1969-01-01"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1968-12-31"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_IN, "[1969-01-01, 1969-01-02]"); + assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE); + + Long anotherDate = (long) Literal.of("1969-01-02T00:00:00.00000").to(TYPE).value(); + assertProjectionStrict(spec, notIn("timestamp", date, anotherDate), + Expression.Operation.NOT_IN, "[1969-01-01, 1969-01-02, 1969-01-03]"); + assertProjectionStrictValue(spec, in("timestamp", date, anotherDate), Expression.Operation.FALSE); + } + @Test public void testDayStrictUpperBound() { Long date = (long) Literal.of("2017-12-01T23:59:59.999999").to(TYPE).value(); @@ -214,6 +341,24 @@ public void testDayStrictUpperBound() { assertProjectionStrictValue(spec, in("timestamp", date, anotherDate), Expression.Operation.FALSE); } + @Test + public void testNegativeDayStrictUpperBound() { + Long date = (long) Literal.of("1969-12-31T23:59:59.999999").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); + + assertProjectionStrictValue(spec, lessThan("timestamp", date), Expression.Operation.FALSE); + assertProjectionStrictValue(spec, lessThanOrEqual("timestamp", date), Expression.Operation.FALSE); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1969-12-31"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1969-12-31"); + assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_IN, "[1969-12-31, 1970-01-01]"); + assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE); + + Long anotherDate = (long) Literal.of("1970-01-01T00:00:00.00000").to(TYPE).value(); + assertProjectionStrict(spec, notIn("timestamp", date, anotherDate), + Expression.Operation.NOT_IN, "[1969-12-31, 1970-01-01]"); + assertProjectionStrictValue(spec, in("timestamp", date, anotherDate), Expression.Operation.FALSE); + } + @Test public void testDayInclusiveLowerBound() { Long date = (long) Literal.of("2017-12-01T00:00:00.00000").to(TYPE).value(); @@ -232,6 +377,24 @@ public void testDayInclusiveLowerBound() { assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE); } + @Test + public void testNegativeDayInclusiveLowerBound() { + Long date = (long) Literal.of("1969-01-01T00:00:00.00000").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); + + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "1969-01-01"); + assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "1969-01-02"); + assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "1969-01-01"); + assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "1969-01-01"); + assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.IN, "[1969-01-01, 1969-01-02]"); + assertProjectionInclusiveValue(spec, notEqual("timestamp", date), Expression.Operation.TRUE); + + Long anotherDate = (long) Literal.of("1969-01-02T00:00:00.00000").to(TYPE).value(); + assertProjectionInclusive(spec, in("timestamp", date, anotherDate), + Expression.Operation.IN, "[1969-01-01, 1969-01-02, 1969-01-03]"); + assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE); + } + @Test public void testDayInclusiveUpperBound() { Long date = (long) Literal.of("2017-12-01T23:59:59.999999").to(TYPE).value(); @@ -250,6 +413,24 @@ public void testDayInclusiveUpperBound() { assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE); } + @Test + public void testNegativeDayInclusiveUpperBound() { + Long date = (long) Literal.of("1969-12-31T23:59:59.999999").to(TYPE).value(); + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build(); + + assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "1970-01-01"); + assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "1970-01-01"); + assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "1970-01-01"); + assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "1969-12-31"); + assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.IN, "[1969-12-31, 1970-01-01]"); + assertProjectionInclusiveValue(spec, notEqual("timestamp", date), Expression.Operation.TRUE); + + Long anotherDate = (long) Literal.of("1970-01-01T00:00:00.00000").to(TYPE).value(); + assertProjectionInclusive(spec, in("timestamp", date, anotherDate), + Expression.Operation.IN, "[1969-12-31, 1970-01-01]"); + assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE); + } + @Test public void testYearStrictLowerBound() { Long date = (long) Literal.of("2017-01-01T00:00:00.00000").to(TYPE).value(); From 146ff9e1aebe19d40ca76a849d39fd3432919ae2 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Tue, 29 Dec 2020 11:46:36 -0800 Subject: [PATCH 4/6] Add docs for the projection fix methods. --- .../iceberg/transforms/ProjectionUtil.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java index 789facc898b7..df2575ce7c2b 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java +++ b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java @@ -258,6 +258,15 @@ static UnboundPredicate transformSet(String fieldName, Iterables.transform(predicate.asSetPredicate().literalSet(), transform::apply)); } + /** + * Fixes an inclusive projection to account for incorrectly transformed values. + *

+ * A bug in 0.10.0 and earlier caused negative values to be incorrectly transformed by date and timestamp transforms + * to 1 larger than the correct value. For example, day(1969-12-31 10:00:00) produced 0 instead of -1. To read data + * written by versions with this bug, this method adjusts the inclusive projection. The current inclusive projection + * is correct, so this modifies the "correct" projection when needed. For example, < day(1969-12-31 10:00:00) will + * produce <= -1 (= 1969-12-31) and is adjusted to <= 0 (= 1969-01-01) because the incorrect transformed value was 0. + */ static UnboundPredicate fixInclusiveTimeProjection(UnboundPredicate projected) { if (projected == null) { return projected; @@ -320,6 +329,13 @@ static UnboundPredicate fixInclusiveTimeProjection(UnboundPredicate + * A bug in 0.10.0 and earlier caused negative values to be incorrectly transformed by date and timestamp transforms + * to 1 larger than the correct value. For example, day(1969-12-31 10:00:00) produced 0 instead of -1. To read data + * written by versions with this bug, this method adjusts the strict projection. + */ static UnboundPredicate fixStrictTimeProjection(UnboundPredicate projected) { if (projected == null) { return null; From 816218928c7fbcddd39b24e5b9e6d675ab6521b0 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Tue, 29 Dec 2020 12:12:25 -0800 Subject: [PATCH 5/6] Fix strict GT and GT_EQ projections. --- .../iceberg/transforms/ProjectionUtil.java | 15 ++++++++++++-- .../transforms/TestDatesProjection.java | 20 +++++++++---------- .../transforms/TestTimestampsProjection.java | 20 +++++++++---------- 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java index df2575ce7c2b..70e4b6747f98 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java +++ b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java @@ -354,9 +354,20 @@ static UnboundPredicate fixStrictTimeProjection(UnboundPredicate= month(1969-11-31) is > -2, but + // 1969-10-31 was previously transformed to month -2 instead of -3. This must use the more strict value. + if (projected.literal().value() <= 0) { + return Expressions.greaterThan(projected.term(), projected.literal().value() + 1); + } + + return projected; + case GT_EQ: - // EQ and GT_EQ do not need to be adjusted because the incorrect value is more strict than the projection - // for example, if the correct strict projection is x > 5, the incorrect value, x > 6, is more strict + if (projected.literal().value() <= 0) { + return Expressions.greaterThanOrEqual(projected.term(), projected.literal().value() + 1); + } + return projected; case EQ: diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java index 15830d6583e2..f674e34b179c 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java @@ -114,8 +114,8 @@ public void testMonthStrictEpoch() { // the boundary cannot be projected because fixing strict projection must fix cases where value + 1 = 0 assertProjectionStrictValue(spec, lessThan("date", date), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThanOrEqual("date", date), Expression.Operation.FALSE); - assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1970-01"); - assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1969-12"); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1970-02"); + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1970-01"); assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "1970-01"); assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); @@ -168,8 +168,8 @@ public void testNegativeMonthStrictLowerBound() { assertProjectionStrictValue(spec, lessThan("date", date), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThanOrEqual("date", date), Expression.Operation.FALSE); - assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1969-01"); - assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1968-12"); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1969-02"); + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1969-01"); assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_IN, "[1969-01, 1969-02]"); assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); @@ -204,8 +204,8 @@ public void testNegativeMonthStrictUpperBound() { assertProjectionStrictValue(spec, lessThan("date", date), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThanOrEqual("date", date), Expression.Operation.FALSE); - assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1969-12"); - assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1969-12"); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1970-01"); + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1970-01"); assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_IN, "[1969-12, 1970-01]"); assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); @@ -389,8 +389,8 @@ public void testNegativeYearStrictLowerBound() { // the boundary cannot be projected because fixing strict projection must fix cases where value + 1 = 0 assertProjectionStrictValue(spec, lessThan("date", date), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThanOrEqual("date", date), Expression.Operation.FALSE); - assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1970"); - assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1969"); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1971"); + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1970"); assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "1970"); assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); @@ -425,8 +425,8 @@ public void testNegativeYearStrictUpperBound() { assertProjectionStrictValue(spec, lessThan("date", date), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThanOrEqual("date", date), Expression.Operation.FALSE); - assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1969"); - assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1969"); + assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "1970"); + assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "1970"); assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_IN, "[1969, 1970]"); assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java index 0237219566b7..36aff6039de4 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java @@ -114,8 +114,8 @@ public void testDayStrictEpoch() { // the boundary cannot be projected because fixing strict projection must fix cases where value + 1 = 0 assertProjectionStrictValue(spec, lessThan("timestamp", date), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThanOrEqual("timestamp", date), Expression.Operation.FALSE); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1970-01-01"); - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1969-12-31"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1970-01-02"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1970-01-01"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "1970-01-01"); assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE); @@ -168,8 +168,8 @@ public void testNegativeMonthStrictLowerBound() { assertProjectionStrictValue(spec, lessThan("timestamp", date), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThanOrEqual("timestamp", date), Expression.Operation.FALSE); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1969-01"); - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1968-12"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1969-02"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1969-01"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_IN, "[1969-01, 1969-02]"); assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE); @@ -204,8 +204,8 @@ public void testNegativeMonthStrictUpperBound() { assertProjectionStrictValue(spec, lessThan("timestamp", date), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThanOrEqual("timestamp", date), Expression.Operation.FALSE); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1969-12"); - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1969-12"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1970-01"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1970-01"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_IN, "[1969-12, 1970-01]"); assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE); @@ -312,8 +312,8 @@ public void testNegativeDayStrictLowerBound() { assertProjectionStrictValue(spec, lessThan("timestamp", date), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThanOrEqual("timestamp", date), Expression.Operation.FALSE); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1969-01-01"); - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1968-12-31"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1969-01-02"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1969-01-01"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_IN, "[1969-01-01, 1969-01-02]"); assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE); @@ -348,8 +348,8 @@ public void testNegativeDayStrictUpperBound() { assertProjectionStrictValue(spec, lessThan("timestamp", date), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThanOrEqual("timestamp", date), Expression.Operation.FALSE); - assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1969-12-31"); - assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1969-12-31"); + assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "1970-01-01"); + assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "1970-01-01"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_IN, "[1969-12-31, 1970-01-01]"); assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE); From 98a9f9b577e62668accda6a109e75eea3d71dcdd Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Tue, 29 Dec 2020 12:30:19 -0800 Subject: [PATCH 6/6] Fix LT and LT_EQ strict projection. --- .../iceberg/transforms/ProjectionUtil.java | 8 +------ .../transforms/TestDatesProjection.java | 22 +++++++++---------- .../transforms/TestTimestampsProjection.java | 21 +++++++++--------- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java index 70e4b6747f98..898829242bf9 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java +++ b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java @@ -344,13 +344,7 @@ static UnboundPredicate fixStrictTimeProjection(UnboundPredicate