storage: PCU hangs #2515
Description
Environment details
- Specify the API at the beginning of the title. For example, "BigQuery: ...").
General, Core, and Other are also allowed as types - OS type and version: Linux
- Java version: 17
- version(s): 2.37.0
Steps to reproduce
I wanted to try out the new parallel composite support, but my example below simply hangs. The program completes with .setAllowParallelCompositeUpload(false).
I see it created many 16mib intermediate objects in the bucket, though. (That also seems wrong since 4gib/16mib ≫32, the maximum number of composite object components.)
Code example
import com.google.cloud.storage.Storage.BlobWriteOption;
import com.google.cloud.storage.StorageOptions;
import com.google.cloud.storage.transfermanager.ParallelUploadConfig;
import com.google.cloud.storage.transfermanager.TransferManagerConfig;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.List;
class X {
private static final String BUCKET_NAME = "SOME BUCKET";
private static final String PREFIX = "prefix";
/** Will create a large local file at this path. */
private static final String TMP_PATH = "/tmp/x";
public static void main(String[] args) throws Exception {
var path = Path.of(TMP_PATH);
try (var ch = Files.newByteChannel(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
ch.position(4L * 1024L * 1024L * 1024L - 1);
ch.write(ByteBuffer.wrap(new byte[1]));
}
var options = StorageOptions.newBuilder().build();
var tmc =
TransferManagerConfig.newBuilder()
.setStorageOptions(options)
.setAllowParallelCompositeUpload(true)
.build();
try (var tm = tmc.getService()) {
System.out.println(
tm.uploadFiles(
List.of(path),
ParallelUploadConfig.newBuilder()
.setBucketName(BUCKET_NAME)
.setPrefix(PREFIX)
.setWriteOptsPerRequest(List.of(BlobWriteOption.disableGzipContent()))
.build())
.getUploadResults());
}
}
}