From 8ec8b63b4c1473d2e28fe8606a0b7bc6aa918e3e Mon Sep 17 00:00:00 2001 From: Robert Metzger Date: Tue, 4 Aug 2020 16:49:38 +0200 Subject: [PATCH] [FLINK-18643][Azure] Build a Flink snapshot release with the nightly cron-job. This closes #13125 --- tools/azure-pipelines/build-apache-repo.yml | 5 + tools/azure-pipelines/build-nightly-dist.yml | 117 +++++++++++++++++++ tools/ci/deploy_nightly_to_s3.sh | 43 +++++++ tools/deploy_to_maven.sh | 99 ---------------- tools/releasing/deploy_staging_jars.sh | 3 +- 5 files changed, 167 insertions(+), 100 deletions(-) create mode 100644 tools/azure-pipelines/build-nightly-dist.yml create mode 100755 tools/ci/deploy_nightly_to_s3.sh delete mode 100755 tools/deploy_to_maven.sh diff --git a/tools/azure-pipelines/build-apache-repo.yml b/tools/azure-pipelines/build-apache-repo.yml index 805ba25288d..4a1f007d1e7 100644 --- a/tools/azure-pipelines/build-apache-repo.yml +++ b/tools/azure-pipelines/build-apache-repo.yml @@ -74,6 +74,11 @@ stages: dependsOn: [] # depending on an empty array makes the stages run in parallel condition: or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['MODE'], 'nightly')) jobs: + - template: build-nightly-dist.yml + parameters: + stage_name: cron_snapshot_deployment + environment: PROFILE="" + container: flink-build-container - template: jobs-template.yml parameters: stage_name: cron_azure diff --git a/tools/azure-pipelines/build-nightly-dist.yml b/tools/azure-pipelines/build-nightly-dist.yml new file mode 100644 index 00000000000..867605c83bf --- /dev/null +++ b/tools/azure-pipelines/build-nightly-dist.yml @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +jobs: + - job: ${{parameters.stage_name}}_binary + pool: + vmImage: 'ubuntu-16.04' + container: flink-build-container + workspace: + clean: all + steps: + - task: Cache@2 + displayName: Cache Maven local repo + inputs: + key: $(CACHE_KEY) + restoreKeys: $(CACHE_FALLBACK_KEY) + path: $(MAVEN_CACHE_FOLDER) + continueOnError: true + - task: CmdLine@2 + displayName: Build snapshot binary release + inputs: + script: | + source ./tools/ci/maven-utils.sh + setup_maven + run_mvn -version + export MVN="run_mvn" + + export RELEASE_VERSION=$(MVN_RUN_VERBOSE=false run_mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + echo "Determined RELEASE_VERSION as '$RELEASE_VERSION' " + cd tools + MVN_RUN_VERBOSE=true SKIP_GPG=true ./releasing/create_binary_release.sh + echo "Created files:" + find ./releasing/release + cd .. + - task: CmdLine@2 + displayName: Upload artifacts to S3 + inputs: + script: | + source ./tools/ci/deploy_nightly_to_s3.sh + + upload_to_s3 ./tools/releasing/release + env: + ARTIFACTS_S3_BUCKET: $(ARTIFACTS_S3_BUCKET) + ARTIFACTS_AWS_ACCESS_KEY_ID: $(ARTIFACTS_AWS_ACCESS_KEY_ID) + ARTIFACTS_AWS_SECRET_ACCESS_KEY: $(ARTIFACTS_AWS_SECRET_ACCESS_KEY) + # Activate this to publish the binary release as a pipeline artifact on Azure + #- task: PublishPipelineArtifact@1 + # displayName: Upload snapshot binary release + # inputs: + # targetPath: ./tools/releasing/release + # artifact: nightly-release + - job: ${{parameters.stage_name}}_maven + pool: + vmImage: 'ubuntu-16.04' + container: flink-build-container + timeoutInMinutes: 100 # 40 minutes per scala version + 20 buffer + workspace: + clean: all + steps: + - task: Cache@2 + displayName: Cache Maven local repo + inputs: + key: $(CACHE_KEY) + restoreKeys: $(CACHE_FALLBACK_KEY) + path: $(MAVEN_CACHE_FOLDER) + continueOnError: true + # Upload snapshot + - task: CmdLine@2 + displayName: Deploy maven snapshot + inputs: + script: | + source ./tools/ci/maven-utils.sh + setup_maven + run_mvn -version + + cd tools + cat << EOF > deploy-settings.xml + + + + apache.snapshots.https + ${MAVEN_DEPLOY_USER} + ${MAVEN_DEPLOY_PASS} + + + + + google-maven-central + GCS Maven Central mirror + https://maven-central-eu.storage-download.googleapis.com/maven2/ + central + + + + EOF + + export CUSTOM_OPTIONS="-Dgpg.skip -Drat.skip -Dcheckstyle.skip --settings $(pwd)/deploy-settings.xml" + export MVN_RUN_VERBOSE=true + ./releasing/deploy_staging_jars.sh + env: + MAVEN_DEPLOY_USER: $(MAVEN_DEPLOY_USER) + MAVEN_DEPLOY_PASS: $(MAVEN_DEPLOY_PASS) diff --git a/tools/ci/deploy_nightly_to_s3.sh b/tools/ci/deploy_nightly_to_s3.sh new file mode 100755 index 00000000000..ca373bdea83 --- /dev/null +++ b/tools/ci/deploy_nightly_to_s3.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# fail on errors +set -e -x + + +function upload_to_s3() { + local FILES_DIR=$1 + + echo "Installing artifacts deployment script" + export ARTIFACTS_DEST="$HOME/bin/artifacts" + curl -sL https://raw.githubusercontent.com/travis-ci/artifacts/master/install | bash + PATH="$(dirname "$ARTIFACTS_DEST"):$PATH" + + echo "Uploading contents of $FILES_DIR to S3:" + + + artifacts upload \ + --bucket $ARTIFACTS_S3_BUCKET \ + --key $ARTIFACTS_AWS_ACCESS_KEY_ID \ + --secret $ARTIFACTS_AWS_SECRET_ACCESS_KEY \ + --target-paths / $FILES_DIR + +} + + diff --git a/tools/deploy_to_maven.sh b/tools/deploy_to_maven.sh deleted file mode 100755 index abc34c8916e..00000000000 --- a/tools/deploy_to_maven.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -# fail on errors -set -e - -# -# Deploys snapshot builds to Apache's snapshot repository. -# - -function getVersion() { - here="`dirname \"$0\"`" # relative - here="`( cd \"$here\" && pwd )`" # absolutized and normalized - if [ -z "$here" ] ; then - # error; for some reason, the path is not accessible - # to the script (e.g. permissions re-evaled after suid) - exit 1 # fail - fi - flink_home="`dirname \"$here\"`" - cd "$flink_home" - echo `mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -E '^([0-9]+.[0-9]+(.[0-9]+)?(-[a-zA-Z0-9]+)?)$'` -} - -function deploy_to_s3() { - local CURRENT_FLINK_VERSION=$1 - local HD=$2 - - echo "Installing artifacts deployment script" - export ARTIFACTS_DEST="$HOME/bin/artifacts" - curl -sL https://raw.githubusercontent.com/travis-ci/artifacts/master/install | bash - PATH="$(dirname "$ARTIFACTS_DEST"):$PATH" - - echo "Deploying flink version $CURRENT_FLINK_VERSION (hadoop=$HD) to s3:" - mkdir flink-$CURRENT_FLINK_VERSION - cp -r flink-dist/target/flink-*-bin/flink-*/* flink-$CURRENT_FLINK_VERSION/ - tar -czf flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz flink-$CURRENT_FLINK_VERSION - - artifacts upload \ - --bucket $ARTIFACTS_S3_BUCKET \ - --key $ARTIFACTS_AWS_ACCESS_KEY_ID \ - --secret $ARTIFACTS_AWS_SECRET_ACCESS_KEY \ - --target-paths / \ - flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz - - # delete files again - rm -rf flink-$CURRENT_FLINK_VERSION - rm flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz -} - -pwd - - -echo "install lifecycle mapping fake plugin" -git clone https://github.com/mfriedenhagen/dummy-lifecycle-mapping-plugin.git -cd dummy-lifecycle-mapping-plugin -mvn -B install -cd .. -rm -rf dummy-lifecycle-mapping-plugin - - -CURRENT_FLINK_VERSION=`getVersion` - -echo "detected current version as: '$CURRENT_FLINK_VERSION'" - -# -# This script deploys our project to sonatype SNAPSHOTS. -# It will deploy a hadoop v2 (yarn) artifact -# - -if [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then - MVN_SNAPSHOT_OPTS="-B -Pdocs-and-source -DskipTests -Drat.skip=true -Drat.ignoreErrors=true -Dcheckstyle.skip=true \ - -DretryFailedDeploymentCount=10 clean deploy" - - # hadoop2 scala 2.11 - echo "deploy standard version (hadoop2) for scala 2.11" - mvn ${MVN_SNAPSHOT_OPTS} - deploy_to_s3 $CURRENT_FLINK_VERSION "hadoop2" - - exit 0 -else - exit 1 -fi - diff --git a/tools/releasing/deploy_staging_jars.sh b/tools/releasing/deploy_staging_jars.sh index 00bf7df353a..6a7af336657 100755 --- a/tools/releasing/deploy_staging_jars.sh +++ b/tools/releasing/deploy_staging_jars.sh @@ -21,6 +21,7 @@ ## Variables with defaults (if not overwritten by environment) ## MVN=${MVN:-mvn} +CUSTOM_OPTIONS=${CUSTOM_OPTIONS:-} # fail immediately set -o errexit @@ -40,7 +41,7 @@ cd .. echo "Deploying to repository.apache.org" -COMMON_OPTIONS="-Prelease,docs-and-source -DskipTests -DretryFailedDeploymentCount=10" +COMMON_OPTIONS="-Prelease,docs-and-source -DskipTests -DretryFailedDeploymentCount=10 $CUSTOM_OPTIONS" echo "Deploying Scala 2.11 version" $MVN clean deploy $COMMON_OPTIONS -Dscala-2.11