Skip to content

Commit 2f3cbe3

Browse files
PhongChuongPhongChuong
authored
feat: add columnNameCharacterMap to LoadJobConfiguration (#3356)
* feat: add columnNameCharacterMap to LoadJobConfiguration * fix: unintentional empty line delete * fix: lint/format
1 parent 7b7e52b commit 2f3cbe3

File tree

3 files changed

+103
-0
lines changed

3 files changed

+103
-0
lines changed

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load
3939

4040
private final List<String> sourceUris;
4141
private final String fileSetSpecType;
42+
private final String columnNameCharacterMap;
4243
private final TableId destinationTable;
4344
private final List<String> decimalTargetTypes;
4445
private final EncryptionConfiguration destinationEncryptionConfiguration;
@@ -69,6 +70,8 @@ public static final class Builder extends JobConfiguration.Builder<LoadJobConfig
6970

7071
private List<String> sourceUris;
7172
private String fileSetSpecType;
73+
private String columnNameCharacterMap;
74+
7275
private TableId destinationTable;
7376
private List<String> decimalTargetTypes;
7477
private EncryptionConfiguration destinationEncryptionConfiguration;
@@ -110,6 +113,7 @@ private Builder(LoadJobConfiguration loadConfiguration) {
110113
this.ignoreUnknownValues = loadConfiguration.ignoreUnknownValues;
111114
this.sourceUris = loadConfiguration.sourceUris;
112115
this.fileSetSpecType = loadConfiguration.fileSetSpecType;
116+
this.columnNameCharacterMap = loadConfiguration.columnNameCharacterMap;
113117
this.schemaUpdateOptions = loadConfiguration.schemaUpdateOptions;
114118
this.autodetect = loadConfiguration.autodetect;
115119
this.destinationEncryptionConfiguration =
@@ -181,6 +185,9 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
181185
if (loadConfigurationPb.getFileSetSpecType() != null) {
182186
this.fileSetSpecType = loadConfigurationPb.getFileSetSpecType();
183187
}
188+
if (loadConfigurationPb.getColumnNameCharacterMap() != null) {
189+
this.columnNameCharacterMap = loadConfigurationPb.getColumnNameCharacterMap();
190+
}
184191
if (loadConfigurationPb.getSchemaUpdateOptions() != null) {
185192
ImmutableList.Builder<JobInfo.SchemaUpdateOption> schemaUpdateOptionsBuilder =
186193
new ImmutableList.Builder<>();
@@ -323,6 +330,20 @@ public Builder setFileSetSpecType(String fileSetSpecType) {
323330
return this;
324331
}
325332

333+
/**
334+
* [Optional] Character map supported for column names in CSV/Parquet loads. Defaults to STRICT
335+
* and can be overridden by Project Config Service. Using this option with unsupporting load
336+
* formats will result in an error.
337+
*
338+
* @see <a
339+
* href="https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#columnnamecharactermap">
340+
* ColumnNameCharacterMap</a>
341+
*/
342+
public Builder setColumnNameCharacterMap(String columnNameCharacterMap) {
343+
this.columnNameCharacterMap = columnNameCharacterMap;
344+
return this;
345+
}
346+
326347
/**
327348
* Defines the list of possible SQL data types to which the source decimal values are converted.
328349
* This list and the precision and the scale parameters of the decimal field determine the
@@ -421,6 +442,7 @@ private LoadJobConfiguration(Builder builder) {
421442
super(builder);
422443
this.sourceUris = builder.sourceUris;
423444
this.fileSetSpecType = builder.fileSetSpecType;
445+
this.columnNameCharacterMap = builder.columnNameCharacterMap;
424446
this.destinationTable = builder.destinationTable;
425447
this.decimalTargetTypes = builder.decimalTargetTypes;
426448
this.createDisposition = builder.createDisposition;
@@ -519,6 +541,17 @@ public String getFileSetSpecType() {
519541
return fileSetSpecType;
520542
}
521543

544+
/**
545+
* Returns the column name character map used in CSV/Parquet loads.
546+
*
547+
* @see <a
548+
* href="https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#columnnamecharactermap">
549+
* ColumnNameCharacterMap</a>
550+
*/
551+
public String getColumnNameCharacterMap() {
552+
return columnNameCharacterMap;
553+
}
554+
522555
public List<String> getDecimalTargetTypes() {
523556
return decimalTargetTypes;
524557
}
@@ -598,6 +631,7 @@ ToStringHelper toStringHelper() {
598631
.add("ignoreUnknownValue", ignoreUnknownValues)
599632
.add("sourceUris", sourceUris)
600633
.add("fileSetSpecType", fileSetSpecType)
634+
.add("columnNameCharacterMap", columnNameCharacterMap)
601635
.add("schemaUpdateOptions", schemaUpdateOptions)
602636
.add("autodetect", autodetect)
603637
.add("timePartitioning", timePartitioning)
@@ -681,6 +715,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
681715
if (fileSetSpecType != null) {
682716
loadConfigurationPb.setFileSetSpecType(fileSetSpecType);
683717
}
718+
if (columnNameCharacterMap != null) {
719+
loadConfigurationPb.setColumnNameCharacterMap(columnNameCharacterMap);
720+
}
684721
if (decimalTargetTypes != null) {
685722
loadConfigurationPb.setDecimalTargetTypes(ImmutableList.copyOf(decimalTargetTypes));
686723
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ public class LoadJobConfigurationTest {
9292
.setWriteDisposition(WRITE_DISPOSITION)
9393
.setFormatOptions(CSV_OPTIONS)
9494
.setFileSetSpecType("FILE_SET_SPEC_TYPE_FILE_SYSTEM_MATCH")
95+
.setColumnNameCharacterMap("STRICT")
9596
.setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES)
9697
.setMaxBadRecords(MAX_BAD_RECORDS)
9798
.setSchema(TABLE_SCHEMA)
@@ -242,6 +243,7 @@ private void compareLoadJobConfiguration(
242243
assertEquals(expected, value);
243244
assertEquals(expected.hashCode(), value.hashCode());
244245
assertEquals(expected.getFileSetSpecType(), value.getFileSetSpecType());
246+
assertEquals(expected.getColumnNameCharacterMap(), value.getColumnNameCharacterMap());
245247
assertEquals(expected.toString(), value.toString());
246248
assertEquals(expected.getDestinationTable(), value.getDestinationTable());
247249
assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes());

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -586,6 +586,8 @@ public class ITBigQueryTest {
586586
RangePartitioning.newBuilder().setField("IntegerField").setRange(RANGE).build();
587587
private static final String LOAD_FILE = "load.csv";
588588
private static final String LOAD_FILE_LARGE = "load_large.csv";
589+
590+
private static final String LOAD_FILE_FLEXIBLE_COLUMN_NAME = "load_flexible_column_name.csv";
589591
private static final String JSON_LOAD_FILE = "load.json";
590592
private static final String JSON_LOAD_FILE_BQ_RESULTSET = "load_bq_resultset.json";
591593
private static final String JSON_LOAD_FILE_SIMPLE = "load_simple.json";
@@ -601,6 +603,7 @@ public class ITBigQueryTest {
601603
private static final TableId TABLE_ID_FASTQUERY_BQ_RESULTSET =
602604
TableId.of(DATASET, "fastquery_testing_bq_resultset");
603605
private static final String CSV_CONTENT = "StringValue1\nStringValue2\n";
606+
private static final String CSV_CONTENT_FLEXIBLE_COLUMN = "name,&ampersand\nrow_name,1";
604607

605608
private static final String JSON_CONTENT =
606609
"{"
@@ -1019,6 +1022,11 @@ public static void beforeClass() throws InterruptedException, IOException {
10191022
storage.create(
10201023
BlobInfo.newBuilder(BUCKET, LOAD_FILE).setContentType("text/plain").build(),
10211024
CSV_CONTENT.getBytes(StandardCharsets.UTF_8));
1025+
storage.create(
1026+
BlobInfo.newBuilder(BUCKET, LOAD_FILE_FLEXIBLE_COLUMN_NAME)
1027+
.setContentType("text/plain")
1028+
.build(),
1029+
CSV_CONTENT_FLEXIBLE_COLUMN.getBytes(StandardCharsets.UTF_8));
10221030
storage.create(
10231031
BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE).setContentType("application/json").build(),
10241032
JSON_CONTENT.getBytes(StandardCharsets.UTF_8));
@@ -6934,4 +6942,60 @@ public void testQueryExportStatistics() throws InterruptedException {
69346942
assertEquals(1L, queryStatistics.getExportDataStats().getFileCount().longValue());
69356943
assertEquals(3L, queryStatistics.getExportDataStats().getRowCount().longValue());
69366944
}
6945+
6946+
@Test
6947+
public void testLoadConfigurationFlexibleColumnName() throws InterruptedException {
6948+
// See https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#columnnamecharactermap for
6949+
// mapping.
6950+
6951+
// Test v1 mapping.
6952+
String v1TableName = "flexible_column_name_data_testing_table_v1";
6953+
TableId v1TableId = TableId.of(DATASET, v1TableName);
6954+
try {
6955+
LoadJobConfiguration loadJobConfigurationV1 =
6956+
LoadJobConfiguration.newBuilder(
6957+
v1TableId,
6958+
"gs://" + BUCKET + "/" + LOAD_FILE_FLEXIBLE_COLUMN_NAME,
6959+
FormatOptions.csv())
6960+
.setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
6961+
.setAutodetect(true)
6962+
.setColumnNameCharacterMap("V1")
6963+
.build();
6964+
Job jobV1 = bigquery.create(JobInfo.of(loadJobConfigurationV1));
6965+
jobV1 = jobV1.waitFor();
6966+
assertNull(jobV1.getStatus().getError());
6967+
6968+
Table remoteTableV1 = bigquery.getTable(DATASET, v1TableName);
6969+
assertNotNull(remoteTableV1);
6970+
assertEquals(
6971+
"_ampersand", remoteTableV1.getDefinition().getSchema().getFields().get(1).getName());
6972+
} finally {
6973+
bigquery.delete(v1TableId);
6974+
}
6975+
6976+
// Test v2 mapping.
6977+
String v2TableName = "flexible_column_name_data_testing_table_v2";
6978+
TableId v2TableId = TableId.of(DATASET, v2TableName);
6979+
try {
6980+
LoadJobConfiguration loadJobConfigurationV2 =
6981+
LoadJobConfiguration.newBuilder(
6982+
v2TableId,
6983+
"gs://" + BUCKET + "/" + LOAD_FILE_FLEXIBLE_COLUMN_NAME,
6984+
FormatOptions.csv())
6985+
.setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
6986+
.setAutodetect(true)
6987+
.setColumnNameCharacterMap("V2")
6988+
.build();
6989+
Job jobV2 = bigquery.create(JobInfo.of(loadJobConfigurationV2));
6990+
jobV2 = jobV2.waitFor();
6991+
assertNull(jobV2.getStatus().getError());
6992+
6993+
Table remoteTableV2 = bigquery.getTable(DATASET, v2TableName);
6994+
assertNotNull(remoteTableV2);
6995+
assertEquals(
6996+
"&ampersand", remoteTableV2.getDefinition().getSchema().getFields().get(1).getName());
6997+
} finally {
6998+
bigquery.delete(v2TableId);
6999+
}
7000+
}
69377001
}

0 commit comments

Comments
 (0)