From f6d09c5c0182767243d6628fb257ddd0a93482e4 Mon Sep 17 00:00:00 2001 From: Jayadeep Jayaraman <53975210+jayadeep-jayaraman@users.noreply.github.com> Date: Tue, 4 Jul 2023 13:40:11 +0530 Subject: [PATCH] [FLINK-32511][FileSystems] Upgrade GCS connector to 2.2.15. This closes #22930 * [FLINK-32511] Upgrade GCS connector to 2.2.15 * Update NOTICE file --- .idea/vcs.xml | 30 +++++++++++++++- .../docs/deployment/filesystems/gcs.md | 4 +-- .../docs/deployment/filesystems/gcs.md | 4 +-- flink-filesystems/flink-gs-fs-hadoop/pom.xml | 4 +-- .../src/main/resources/META-INF/NOTICE | 36 +++++++++---------- 5 files changed, 53 insertions(+), 25 deletions(-) diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 165d593076e..8c1dfffdcbb 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -15,6 +15,34 @@ @@ -22,4 +50,4 @@ - + \ No newline at end of file diff --git a/docs/content.zh/docs/deployment/filesystems/gcs.md b/docs/content.zh/docs/deployment/filesystems/gcs.md index 2a92caf9771..363cd2fec90 100644 --- a/docs/content.zh/docs/deployment/filesystems/gcs.md +++ b/docs/content.zh/docs/deployment/filesystems/gcs.md @@ -55,7 +55,7 @@ Note that these examples are *not* exhaustive and you can use GCS in other place Flink provides the `flink-gs-fs-hadoop` file system to write to GCS. This implementation is self-contained with no dependency footprint, so there is no need to add Hadoop to the classpath to use it. -`flink-gs-fs-hadoop` registers a `FileSystem` wrapper for URIs with the *gs://* scheme. It uses Google's [gcs-connector](https://mvnrepository.com/artifact/com.google.cloud.bigdataoss/gcs-connector/hadoop3-2.2.11) Hadoop library to access GCS. It also uses Google's [google-cloud-storage](https://mvnrepository.com/artifact/com.google.cloud/google-cloud-storage/2.15.0) library to provide `RecoverableWriter` support. +`flink-gs-fs-hadoop` registers a `FileSystem` wrapper for URIs with the *gs://* scheme. It uses Google's [gcs-connector](https://mvnrepository.com/artifact/com.google.cloud.bigdataoss/gcs-connector/hadoop3-2.2.15) Hadoop library to access GCS. It also uses Google's [google-cloud-storage](https://mvnrepository.com/artifact/com.google.cloud/google-cloud-storage/2.15.0) library to provide `RecoverableWriter` support. This file system can be used with the [FileSystem connector]({{< ref "docs/connectors/datastream/filesystem.md" >}}). @@ -68,7 +68,7 @@ cp ./opt/flink-gs-fs-hadoop-{{< version >}}.jar ./plugins/gs-fs-hadoop/ ### Configuration -The underlying Hadoop file system can be configured using the [Hadoop configuration keys](https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/v2.2.11/gcs/CONFIGURATION.md) for `gcs-connector` by adding the configurations to your `flink-conf.yaml`. +The underlying Hadoop file system can be configured using the [Hadoop configuration keys](https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/v2.2.15/gcs/CONFIGURATION.md) for `gcs-connector` by adding the configurations to your `flink-conf.yaml`. For example, `gcs-connector` has a `fs.gs.http.connect-timeout` configuration key. If you want to change it, you need to set `gs.http.connect-timeout: xyz` in `flink-conf.yaml`. Flink will internally translate this back to `fs.gs.http.connect-timeout`. diff --git a/docs/content/docs/deployment/filesystems/gcs.md b/docs/content/docs/deployment/filesystems/gcs.md index a7d891f4012..97e008eca33 100644 --- a/docs/content/docs/deployment/filesystems/gcs.md +++ b/docs/content/docs/deployment/filesystems/gcs.md @@ -55,7 +55,7 @@ Note that these examples are *not* exhaustive and you can use GCS in other place Flink provides the `flink-gs-fs-hadoop` file system to write to GCS. This implementation is self-contained with no dependency footprint, so there is no need to add Hadoop to the classpath to use it. -`flink-gs-fs-hadoop` registers a `FileSystem` wrapper for URIs with the *gs://* scheme. It uses Google's [gcs-connector](https://mvnrepository.com/artifact/com.google.cloud.bigdataoss/gcs-connector/hadoop3-2.2.11) Hadoop library to access GCS. It also uses Google's [google-cloud-storage](https://mvnrepository.com/artifact/com.google.cloud/google-cloud-storage/2.15.0) library to provide `RecoverableWriter` support. +`flink-gs-fs-hadoop` registers a `FileSystem` wrapper for URIs with the *gs://* scheme. It uses Google's [gcs-connector](https://mvnrepository.com/artifact/com.google.cloud.bigdataoss/gcs-connector/hadoop3-2.2.15) Hadoop library to access GCS. It also uses Google's [google-cloud-storage](https://mvnrepository.com/artifact/com.google.cloud/google-cloud-storage/2.15.0) library to provide `RecoverableWriter` support. This file system can be used with the [FileSystem connector]({{< ref "docs/connectors/datastream/filesystem.md" >}}). @@ -68,7 +68,7 @@ cp ./opt/flink-gs-fs-hadoop-{{< version >}}.jar ./plugins/gs-fs-hadoop/ ### Configuration -The underlying Hadoop file system can be configured using the [Hadoop configuration keys](https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/v2.2.11/gcs/CONFIGURATION.md) for `gcs-connector` by adding the configurations to your `flink-conf.yaml`. +The underlying Hadoop file system can be configured using the [Hadoop configuration keys](https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/v2.2.15/gcs/CONFIGURATION.md) for `gcs-connector` by adding the configurations to your `flink-conf.yaml`. For example, `gcs-connector` has a `fs.gs.http.connect-timeout` configuration key. If you want to change it, you need to set `gs.http.connect-timeout: xyz` in `flink-conf.yaml`. Flink will internally translate this back to `fs.gs.http.connect-timeout`. diff --git a/flink-filesystems/flink-gs-fs-hadoop/pom.xml b/flink-filesystems/flink-gs-fs-hadoop/pom.xml index 2563dfd3415..623f7068d56 100644 --- a/flink-filesystems/flink-gs-fs-hadoop/pom.xml +++ b/flink-filesystems/flink-gs-fs-hadoop/pom.xml @@ -35,9 +35,9 @@ under the License. 2.15.0 - hadoop3-2.2.11 + hadoop3-2.2.15 - 1.50.2 + 1.50.3 diff --git a/flink-filesystems/flink-gs-fs-hadoop/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-gs-fs-hadoop/src/main/resources/META-INF/NOTICE index 814fd382dc3..7a34d1b4ef6 100644 --- a/flink-filesystems/flink-gs-fs-hadoop/src/main/resources/META-INF/NOTICE +++ b/flink-filesystems/flink-gs-fs-hadoop/src/main/resources/META-INF/NOTICE @@ -20,10 +20,10 @@ This project bundles the following dependencies under the Apache Software Licens - com.google.apis:google-api-services-iamcredentials:v1-rev20211203-2.0.0 - com.google.apis:google-api-services-storage:v1-rev20220705-2.0.0 - com.google.auto.value:auto-value-annotations:1.10 -- com.google.cloud.bigdataoss:gcs-connector:hadoop3-2.2.11 -- com.google.cloud.bigdataoss:gcsio:2.2.11 -- com.google.cloud.bigdataoss:util-hadoop:hadoop3-2.2.11 -- com.google.cloud.bigdataoss:util:2.2.11 +- com.google.cloud.bigdataoss:gcs-connector:hadoop3-2.2.15 +- com.google.cloud.bigdataoss:gcsio:2.2.15 +- com.google.cloud.bigdataoss:util-hadoop:hadoop3-2.2.15 +- com.google.cloud.bigdataoss:util:2.2.15 - com.google.cloud:google-cloud-core-grpc:2.8.27 - com.google.cloud:google-cloud-core-http:2.8.27 - com.google.cloud:google-cloud-core:2.8.27 @@ -42,20 +42,20 @@ This project bundles the following dependencies under the Apache Software Licens - com.google.oauth-client:google-oauth-client:1.34.1 - com.lmax:disruptor:3.4.2 - commons-codec:commons-codec:1.15 -- io.grpc:grpc-alts:1.50.2 -- io.grpc:grpc-api:1.50.2 -- io.grpc:grpc-auth:1.50.2 -- io.grpc:grpc-census:1.50.2 -- io.grpc:grpc-context:1.50.2 -- io.grpc:grpc-core:1.50.2 -- io.grpc:grpc-googleapis:1.50.2 -- io.grpc:grpc-grpclb:1.50.2 -- io.grpc:grpc-netty-shaded:1.50.2 -- io.grpc:grpc-protobuf-lite:1.50.2 -- io.grpc:grpc-protobuf:1.50.2 -- io.grpc:grpc-services:1.50.2 -- io.grpc:grpc-stub:1.50.2 -- io.grpc:grpc-xds:1.50.2 +- io.grpc:grpc-alts:1.50.3 +- io.grpc:grpc-api:1.50.3 +- io.grpc:grpc-auth:1.50.3 +- io.grpc:grpc-census:1.50.3 +- io.grpc:grpc-context:1.50.3 +- io.grpc:grpc-core:1.50.3 +- io.grpc:grpc-googleapis:1.50.3 +- io.grpc:grpc-grpclb:1.50.3 +- io.grpc:grpc-netty-shaded:1.50.3 +- io.grpc:grpc-protobuf-lite:1.50.3 +- io.grpc:grpc-protobuf:1.50.3 +- io.grpc:grpc-services:1.50.3 +- io.grpc:grpc-stub:1.50.3 +- io.grpc:grpc-xds:1.50.3 - io.opencensus:opencensus-api:0.31.1 - io.opencensus:opencensus-contrib-exemplar-util:0.31.0 - io.opencensus:opencensus-contrib-grpc-metrics:0.31.0