From ed74144424a4b9ea2d3b2c43fb278a2545e3d969 Mon Sep 17 00:00:00 2001 From: slilichenko Date: Sun, 27 Nov 2022 08:49:51 -0800 Subject: [PATCH 1/8] Improve Integer conversion in BigQueryIO's Storage Write API method. --- .../bigquery/TableRowToStorageApiProto.java | 19 +++++- .../TableRowToStorageApiProtoTest.java | 60 ++++++++++++++++++- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java index eaf90fea82959..d592314331988 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java @@ -36,6 +36,7 @@ import com.google.protobuf.DynamicMessage; import com.google.protobuf.Message; import java.math.BigDecimal; +import java.math.BigInteger; import java.math.RoundingMode; import java.time.Instant; import java.time.LocalDate; @@ -408,9 +409,25 @@ private static void fieldDescriptorFromTableField( case "INT64": case "INTEGER": if (value instanceof String) { - return Long.valueOf((String) value); + try { + return Long.valueOf((String) value); + } catch (NumberFormatException e) { + // Expected. Element will be added to the failed element PCollection + } } else if (value instanceof Integer || value instanceof Long) { return ((Number) value).longValue(); + } else if (value instanceof BigDecimal) { + try { + return ((BigDecimal) value).longValueExact(); + } catch (ArithmeticException e) { + // Expected + } + } else if( value instanceof BigInteger) { + try { + return ((BigInteger) value).longValueExact(); + } catch (ArithmeticException e) { + // Expected + } } break; case "FLOAT64": diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java index 0a376c8082d33..bc28f49f5de50 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import com.google.api.services.bigquery.model.TableCell; import com.google.api.services.bigquery.model.TableFieldSchema; @@ -31,14 +32,18 @@ import com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Label; import com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Type; import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.DynamicMessage; import java.math.BigDecimal; +import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto.SchemaConversionException; +import org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto.SchemaInformation; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Functions; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; @@ -50,10 +55,11 @@ @RunWith(JUnit4.class) @SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) + "nullness" // TODO(https://github.com/apache/beam/issues/20497) }) /** Unit tests for {@link org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto}. */ public class TableRowToStorageApiProtoTest { + // Schemas we test. // The TableRow class has special semantics for fields named "f". To ensure we handel them // properly, we test schemas @@ -818,4 +824,56 @@ public void testNullRepeatedDescriptorFromTableSchema() throws Exception { (List) msg.getField(fieldDescriptors.get("repeatednof2")); assertTrue(repeatednof2.isEmpty()); } + + @Test + public void testIntegerTypeConversion() throws DescriptorValidationException { + String intFieldName = "int_field"; + TableSchema tableSchema = new TableSchema() + .setFields( + ImmutableList.builder() + .add( + new TableFieldSchema() + .setType("INTEGER") + .setName(intFieldName) + .setMode("REQUIRED") + ) + .build()); + TableRowToStorageApiProto.SchemaInformation schemaInformation = + TableRowToStorageApiProto.SchemaInformation.fromTableSchema(tableSchema + ); + SchemaInformation fieldSchema = schemaInformation.getSchemaForField(intFieldName); + Descriptor schemaDescriptor = + TableRowToStorageApiProto.getDescriptorFromTableSchema(tableSchema); + FieldDescriptor fieldDescriptor = schemaDescriptor.findFieldByName(intFieldName); + + Object[] validIntValues = new Object[] { + "123", + 123L, + 123, + new BigDecimal("123"), + new BigInteger("123") + }; + for( Object validValue : validIntValues) { + try { + TableRowToStorageApiProto.singularFieldToProtoValue(fieldSchema, fieldDescriptor, validValue , false); + } catch (SchemaConversionException e) { + fail("Failed to convert value " + validValue + " of type " + validValue.getClass() + " to INTEGER: " + e); + } + } + + Object[] invalidIntValues = new Object[] { + "12.123", // Fractional part + Long.toString(Long.MAX_VALUE) + '0', // String numeric exceeding max value of INT64 + new BigDecimal("12.123"), // Fractional part + new BigInteger(String.valueOf(Long.MAX_VALUE)).add(new BigInteger("10")) // Exceeds max value + }; + for( Object invalidValue : invalidIntValues) { + try { + TableRowToStorageApiProto.singularFieldToProtoValue(fieldSchema, fieldDescriptor, invalidValue , false); + fail("Expected to throw an exception converting " + invalidValue + " of type " + invalidValue.getClass() + " to INTEGER: "); + } catch (SchemaConversionException e) { + // expected exception + } + } + } } From 370f79ef4b73ab8dd533ca557bb7528861349424 Mon Sep 17 00:00:00 2001 From: slilichenko Date: Mon, 28 Nov 2022 19:59:15 -0800 Subject: [PATCH 2/8] Improve Integer conversion in BigQueryIO's Storage Write API method. Introduced a dedicated Exception. Improved error messages by capturing the source failure and reducing message sizes. --- .../bigquery/TableRowToStorageApiProto.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java index d592314331988..0677f3eef8624 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java @@ -82,7 +82,7 @@ public class TableRowToStorageApiProto { .toFormatter() .withZone(ZoneOffset.UTC); - public static class SchemaConversionException extends Exception { + abstract public static class SchemaConversionException extends Exception { SchemaConversionException(String msg) { super(msg); } @@ -108,6 +108,20 @@ public static class SchemaDoesntMatchException extends SchemaConversionException } } + public static class SingleValueConversionException extends SchemaConversionException { + SingleValueConversionException(Object sourceValue, SchemaInformation schema, Exception e) { + super("Column: " + getPrettyFieldName(schema) + " (" + schema.getType() + "). " + + "Value: " + sourceValue + " (" + sourceValue.getClass().getName() + + "). Reason: " + e); + } + + private static String getPrettyFieldName(SchemaInformation schema) { + String fullName = schema.getFullName(); + String rootPrefix = "__root__."; + return fullName.startsWith(rootPrefix) ? fullName.substring(rootPrefix.length()) : fullName; + } + } + static class SchemaInformation { private final TableFieldSchema tableFieldSchema; private final List subFields; @@ -412,7 +426,7 @@ private static void fieldDescriptorFromTableField( try { return Long.valueOf((String) value); } catch (NumberFormatException e) { - // Expected. Element will be added to the failed element PCollection + throw new SingleValueConversionException(value, schemaInformation, e); } } else if (value instanceof Integer || value instanceof Long) { return ((Number) value).longValue(); @@ -420,13 +434,13 @@ private static void fieldDescriptorFromTableField( try { return ((BigDecimal) value).longValueExact(); } catch (ArithmeticException e) { - // Expected + throw new SingleValueConversionException(value, schemaInformation, e); } } else if( value instanceof BigInteger) { try { return ((BigInteger) value).longValueExact(); } catch (ArithmeticException e) { - // Expected + throw new SingleValueConversionException(value, schemaInformation, e); } } break; @@ -573,7 +587,7 @@ private static void fieldDescriptorFromTableField( } throw new SchemaDoesntMatchException( - "Unexpected value :" + "Unexpected value: " + value + ", type: " + (value == null ? "null" : value.getClass()) From 94c17f4d6bdba9074bff36f8318e6c78eaf193e8 Mon Sep 17 00:00:00 2001 From: slilichenko Date: Mon, 28 Nov 2022 20:02:59 -0800 Subject: [PATCH 3/8] Improve Integer conversion in BigQueryIO's Storage Write API method. Introduced a dedicated Exception; improved error messages by capturing the source failure. --- .../TableRowToStorageApiProtoTest.java | 56 +++++++++++++------ 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java index bc28f49f5de50..319fc3f0d6473 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java @@ -846,33 +846,53 @@ public void testIntegerTypeConversion() throws DescriptorValidationException { TableRowToStorageApiProto.getDescriptorFromTableSchema(tableSchema); FieldDescriptor fieldDescriptor = schemaDescriptor.findFieldByName(intFieldName); - Object[] validIntValues = new Object[] { - "123", - 123L, - 123, - new BigDecimal("123"), - new BigInteger("123") + Object[][] validIntValues = new Object[][]{ + // Source and expected converted values. + {"123", 123L}, + {123L, 123L}, + {123, 123L}, + {new BigDecimal("123"), 123L}, + {new BigInteger("123"), 123L} }; - for( Object validValue : validIntValues) { + for (Object[] validValue : validIntValues) { + Object sourceValue = validValue[0]; + Long expectedConvertedValue = (Long) validValue[1]; try { - TableRowToStorageApiProto.singularFieldToProtoValue(fieldSchema, fieldDescriptor, validValue , false); + Object converted = TableRowToStorageApiProto.singularFieldToProtoValue(fieldSchema, + fieldDescriptor, sourceValue, false); + assertEquals(expectedConvertedValue, converted); } catch (SchemaConversionException e) { - fail("Failed to convert value " + validValue + " of type " + validValue.getClass() + " to INTEGER: " + e); + fail("Failed to convert value " + sourceValue + " of type " + validValue.getClass() + + " to INTEGER: " + e); } } - Object[] invalidIntValues = new Object[] { - "12.123", // Fractional part - Long.toString(Long.MAX_VALUE) + '0', // String numeric exceeding max value of INT64 - new BigDecimal("12.123"), // Fractional part - new BigInteger(String.valueOf(Long.MAX_VALUE)).add(new BigInteger("10")) // Exceeds max value + Object[][] invalidIntValues = new Object[][] { + // Value and expected error message + {"12.123", "Column: " + + intFieldName + + " (INTEGER). Value: 12.123 (java.lang.String). Reason: java.lang.NumberFormatException: For input string: \"12.123\""}, + {Long.toString(Long.MAX_VALUE) + '0', "Column: " + + intFieldName + + " (INTEGER). Value: 92233720368547758070 (java.lang.String). Reason: java.lang.NumberFormatException: For input string: \"92233720368547758070\""}, + {new BigDecimal("12.123"), "Column: " + + intFieldName + + " (INTEGER). Value: 12.123 (java.math.BigDecimal). Reason: java.lang.ArithmeticException: Rounding necessary"}, + {new BigInteger(String.valueOf(Long.MAX_VALUE)).add(new BigInteger("10")), "Column: " + + intFieldName + + " (INTEGER). Value: 9223372036854775817 (java.math.BigInteger). Reason: java.lang.ArithmeticException: BigInteger out of long range"} + }; - for( Object invalidValue : invalidIntValues) { + for (Object[] invalidValue : invalidIntValues) { + Object sourceValue = invalidValue[0]; + String expectedError = (String) invalidValue[1]; try { - TableRowToStorageApiProto.singularFieldToProtoValue(fieldSchema, fieldDescriptor, invalidValue , false); - fail("Expected to throw an exception converting " + invalidValue + " of type " + invalidValue.getClass() + " to INTEGER: "); + TableRowToStorageApiProto.singularFieldToProtoValue(fieldSchema, fieldDescriptor, + sourceValue, false); + fail("Expected to throw an exception converting " + sourceValue + " of type " + + invalidValue.getClass() + " to INTEGER: "); } catch (SchemaConversionException e) { - // expected exception + assertEquals("Exception message", expectedError, e.getMessage()); } } } From 74e98337ddc948384cfe1c023cb9b7ac98b4a89f Mon Sep 17 00:00:00 2001 From: slilichenko Date: Mon, 28 Nov 2022 20:04:42 -0800 Subject: [PATCH 4/8] Fixed a minor typo. --- .../beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java index 319fc3f0d6473..8aeb60ff9401f 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java @@ -890,7 +890,7 @@ public void testIntegerTypeConversion() throws DescriptorValidationException { TableRowToStorageApiProto.singularFieldToProtoValue(fieldSchema, fieldDescriptor, sourceValue, false); fail("Expected to throw an exception converting " + sourceValue + " of type " - + invalidValue.getClass() + " to INTEGER: "); + + invalidValue.getClass() + " to INTEGER"); } catch (SchemaConversionException e) { assertEquals("Exception message", expectedError, e.getMessage()); } From 52ffd203e73b89203100ae088141c72fdc525f99 Mon Sep 17 00:00:00 2001 From: slilichenko Date: Sun, 27 Nov 2022 08:49:51 -0800 Subject: [PATCH 5/8] Merged PR with new commits. --- .../sdk/io/gcp/bigquery/TableRowToStorageApiProto.java | 2 +- .../io/gcp/bigquery/TableRowToStorageApiProtoTest.java | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java index 048f1b625a485..4bf872989f15b 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java @@ -118,7 +118,7 @@ public static class SingleValueConversionException extends SchemaConversionExcep private static String getPrettyFieldName(SchemaInformation schema) { String fullName = schema.getFullName(); - String rootPrefix = "__root__."; + String rootPrefix = "root."; return fullName.startsWith(rootPrefix) ? fullName.substring(rootPrefix.length()) : fullName; } } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java index 389a9898cadc1..3fec44649f6b1 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java @@ -846,7 +846,7 @@ public void testIntegerTypeConversion() throws DescriptorValidationException { ); SchemaInformation fieldSchema = schemaInformation.getSchemaForField(intFieldName); Descriptor schemaDescriptor = - TableRowToStorageApiProto.getDescriptorFromTableSchema(tableSchema); + TableRowToStorageApiProto.getDescriptorFromTableSchema(tableSchema, true); FieldDescriptor fieldDescriptor = schemaDescriptor.findFieldByName(intFieldName); Object[][] validIntValues = new Object[][]{ @@ -874,16 +874,16 @@ public void testIntegerTypeConversion() throws DescriptorValidationException { // Value and expected error message {"12.123", "Column: " + intFieldName - + " (INTEGER). Value: 12.123 (java.lang.String). Reason: java.lang.NumberFormatException: For input string: \"12.123\""}, + + " (INT64). Value: 12.123 (java.lang.String). Reason: java.lang.NumberFormatException: For input string: \"12.123\""}, {Long.toString(Long.MAX_VALUE) + '0', "Column: " + intFieldName - + " (INTEGER). Value: 92233720368547758070 (java.lang.String). Reason: java.lang.NumberFormatException: For input string: \"92233720368547758070\""}, + + " (INT64). Value: 92233720368547758070 (java.lang.String). Reason: java.lang.NumberFormatException: For input string: \"92233720368547758070\""}, {new BigDecimal("12.123"), "Column: " + intFieldName - + " (INTEGER). Value: 12.123 (java.math.BigDecimal). Reason: java.lang.ArithmeticException: Rounding necessary"}, + + " (INT64). Value: 12.123 (java.math.BigDecimal). Reason: java.lang.ArithmeticException: Rounding necessary"}, {new BigInteger(String.valueOf(Long.MAX_VALUE)).add(new BigInteger("10")), "Column: " + intFieldName - + " (INTEGER). Value: 9223372036854775817 (java.math.BigInteger). Reason: java.lang.ArithmeticException: BigInteger out of long range"} + + " (INT64). Value: 9223372036854775817 (java.math.BigInteger). Reason: java.lang.ArithmeticException: BigInteger out of long range"} }; for (Object[] invalidValue : invalidIntValues) { From 8c65aa975079e3cd02e79250eb0e7450039ff253 Mon Sep 17 00:00:00 2001 From: Lukasz Cwik Date: Mon, 12 Dec 2022 09:50:25 -0800 Subject: [PATCH 6/8] Update sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java --- .../beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java index 4bf872989f15b..38590d4bba99d 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java @@ -618,7 +618,7 @@ private static void fieldDescriptorFromTableField( } catch (ArithmeticException e) { throw new SingleValueConversionException(value, schemaInformation, e); } - } else if( value instanceof BigInteger) { + } else if (value instanceof BigInteger) { try { return ((BigInteger) value).longValueExact(); } catch (ArithmeticException e) { From 02eade68fa08538b75a6eaae6a67c67491d1d82a Mon Sep 17 00:00:00 2001 From: Lukasz Cwik Date: Mon, 12 Dec 2022 14:12:34 -0800 Subject: [PATCH 7/8] Apply suggestions from code review Apply spotless --- .../bigquery/TableRowToStorageApiProto.java | 13 +- .../TableRowToStorageApiProtoTest.java | 112 +++++++++++------- 2 files changed, 79 insertions(+), 46 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java index 38590d4bba99d..3cf9582435577 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java @@ -111,7 +111,18 @@ public static class SchemaDoesntMatchException extends SchemaConversionException public static class SingleValueConversionException extends SchemaConversionException { SingleValueConversionException(Object sourceValue, SchemaInformation schema, Exception e) { - super("Column: " + getPrettyFieldName(schema) + " (" + schema.getType() + "). " + super( + "Column: " + + getPrettyFieldName(schema) + + " (" + + schema.getType() + + "). " + + "Value: " + + sourceValue + + " (" + + sourceValue.getClass().getName() + + "). Reason: " + + e); + "Value: " + sourceValue + " (" + sourceValue.getClass().getName() + "). Reason: " + e); } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java index 3fec44649f6b1..8be2c9db33423 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java @@ -57,7 +57,7 @@ @RunWith(JUnit4.class) @SuppressWarnings({ - "nullness" // TODO(https://github.com/apache/beam/issues/20497) + "nullness" // TODO(https://github.com/apache/beam/issues/20497) }) /** Unit tests for {@link org.apache.beam.sdk.io.gcp.bigquery.TableRowToStorageApiProto}. */ public class TableRowToStorageApiProtoTest { @@ -831,69 +831,91 @@ public void testNullRepeatedDescriptorFromTableSchema() throws Exception { @Test public void testIntegerTypeConversion() throws DescriptorValidationException { String intFieldName = "int_field"; - TableSchema tableSchema = new TableSchema() - .setFields( - ImmutableList.builder() - .add( - new TableFieldSchema() - .setType("INTEGER") - .setName(intFieldName) - .setMode("REQUIRED") - ) - .build()); + TableSchema tableSchema = + new TableSchema() + .setFields( + ImmutableList.builder() + .add( + new TableFieldSchema() + .setType("INTEGER") + .setName(intFieldName) + .setMode("REQUIRED")) + .build()); TableRowToStorageApiProto.SchemaInformation schemaInformation = - TableRowToStorageApiProto.SchemaInformation.fromTableSchema(tableSchema - ); + TableRowToStorageApiProto.SchemaInformation.fromTableSchema(tableSchema); SchemaInformation fieldSchema = schemaInformation.getSchemaForField(intFieldName); Descriptor schemaDescriptor = TableRowToStorageApiProto.getDescriptorFromTableSchema(tableSchema, true); FieldDescriptor fieldDescriptor = schemaDescriptor.findFieldByName(intFieldName); - Object[][] validIntValues = new Object[][]{ - // Source and expected converted values. - {"123", 123L}, - {123L, 123L}, - {123, 123L}, - {new BigDecimal("123"), 123L}, - {new BigInteger("123"), 123L} - }; + Object[][] validIntValues = + new Object[][] { + // Source and expected converted values. + {"123", 123L}, + {123L, 123L}, + {123, 123L}, + {new BigDecimal("123"), 123L}, + {new BigInteger("123"), 123L} + }; for (Object[] validValue : validIntValues) { Object sourceValue = validValue[0]; Long expectedConvertedValue = (Long) validValue[1]; try { - Object converted = TableRowToStorageApiProto.singularFieldToProtoValue(fieldSchema, - fieldDescriptor, sourceValue, false); + Object converted = + TableRowToStorageApiProto.singularFieldToProtoValue( + fieldSchema, fieldDescriptor, sourceValue, false); assertEquals(expectedConvertedValue, converted); } catch (SchemaConversionException e) { - fail("Failed to convert value " + sourceValue + " of type " + validValue.getClass() - + " to INTEGER: " + e); + fail( + "Failed to convert value " + + sourceValue + + " of type " + + validValue.getClass() + + " to INTEGER: " + + e); } } - Object[][] invalidIntValues = new Object[][] { - // Value and expected error message - {"12.123", "Column: " - + intFieldName - + " (INT64). Value: 12.123 (java.lang.String). Reason: java.lang.NumberFormatException: For input string: \"12.123\""}, - {Long.toString(Long.MAX_VALUE) + '0', "Column: " - + intFieldName - + " (INT64). Value: 92233720368547758070 (java.lang.String). Reason: java.lang.NumberFormatException: For input string: \"92233720368547758070\""}, - {new BigDecimal("12.123"), "Column: " - + intFieldName - + " (INT64). Value: 12.123 (java.math.BigDecimal). Reason: java.lang.ArithmeticException: Rounding necessary"}, - {new BigInteger(String.valueOf(Long.MAX_VALUE)).add(new BigInteger("10")), "Column: " - + intFieldName - + " (INT64). Value: 9223372036854775817 (java.math.BigInteger). Reason: java.lang.ArithmeticException: BigInteger out of long range"} - - }; + Object[][] invalidIntValues = + new Object[][] { + // Value and expected error message + { + "12.123", + "Column: " + + intFieldName + + " (INT64). Value: 12.123 (java.lang.String). Reason: java.lang.NumberFormatException: For input string: \"12.123\"" + }, + { + Long.toString(Long.MAX_VALUE) + '0', + "Column: " + + intFieldName + + " (INT64). Value: 92233720368547758070 (java.lang.String). Reason: java.lang.NumberFormatException: For input string: \"92233720368547758070\"" + }, + { + new BigDecimal("12.123"), + "Column: " + + intFieldName + + " (INT64). Value: 12.123 (java.math.BigDecimal). Reason: java.lang.ArithmeticException: Rounding necessary" + }, + { + new BigInteger(String.valueOf(Long.MAX_VALUE)).add(new BigInteger("10")), + "Column: " + + intFieldName + + " (INT64). Value: 9223372036854775817 (java.math.BigInteger). Reason: java.lang.ArithmeticException: BigInteger out of long range" + } + }; for (Object[] invalidValue : invalidIntValues) { Object sourceValue = invalidValue[0]; String expectedError = (String) invalidValue[1]; try { - TableRowToStorageApiProto.singularFieldToProtoValue(fieldSchema, fieldDescriptor, - sourceValue, false); - fail("Expected to throw an exception converting " + sourceValue + " of type " - + invalidValue.getClass() + " to INTEGER"); + TableRowToStorageApiProto.singularFieldToProtoValue( + fieldSchema, fieldDescriptor, sourceValue, false); + fail( + "Expected to throw an exception converting " + + sourceValue + + " of type " + + invalidValue.getClass() + + " to INTEGER"); } catch (SchemaConversionException e) { assertEquals("Exception message", expectedError, e.getMessage()); } From 40260960aab7804eed21447794fd83650f551374 Mon Sep 17 00:00:00 2001 From: Lukasz Cwik Date: Mon, 12 Dec 2022 15:58:40 -0800 Subject: [PATCH 8/8] Update sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java --- .../beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java index 3cf9582435577..e860f7b738854 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProto.java @@ -123,8 +123,6 @@ public static class SingleValueConversionException extends SchemaConversionExcep + sourceValue.getClass().getName() + "). Reason: " + e); - + "Value: " + sourceValue + " (" + sourceValue.getClass().getName() - + "). Reason: " + e); } private static String getPrettyFieldName(SchemaInformation schema) {