From d08356a1afe9a86b06c3978c04b0ef467d1685a6 Mon Sep 17 00:00:00 2001 From: Chesnay Schepler Date: Fri, 11 Aug 2023 10:13:04 +0200 Subject: [PATCH] [FLINK-32834] Write all tmp files to /tmp/ - prevent files from being picked up by git/rat - don't use /target because mvn clean would interfere - tmp dirs under tools/ci would be neat, but we lack a central place to create it --- tools/ci/compile.sh | 14 +++++++++----- tools/ci/shade.sh | 30 +++++++++++++++-------------- tools/ci/verify_bundled_optional.sh | 2 +- tools/ci/verify_scala_suffixes.sh | 2 +- 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/tools/ci/compile.sh b/tools/ci/compile.sh index 92b8d331755..49743999707 100755 --- a/tools/ci/compile.sh +++ b/tools/ci/compile.sh @@ -70,24 +70,28 @@ fi echo "============ Checking Javadocs ============" +javadoc_output=/tmp/javadoc.out + # use the same invocation as .github/workflows/docs.sh -run_mvn javadoc:aggregate -DadditionalJOption='-Xdoclint:none' \ +$MVN javadoc:aggregate -DadditionalJOption='-Xdoclint:none' \ -Dmaven.javadoc.failOnError=false -Dcheckstyle.skip=true -Denforcer.skip=true -Dspotless.skip=true -Drat.skip=true \ - -Dheader=someTestHeader > javadoc.out + -Dheader=someTestHeader > ${javadoc_output} EXIT_CODE=$? if [ $EXIT_CODE != 0 ] ; then echo "ERROR in Javadocs. Printing full output:" - cat javadoc.out ; rm javadoc.out + cat ${javadoc_output} exit $EXIT_CODE fi echo "============ Checking Scaladocs ============" -run_mvn scala:doc -Dcheckstyle.skip=true -Denforcer.skip=true -Dspotless.skip=true -pl flink-scala 2> scaladoc.out +scaladoc_output=/tmp/scaladoc.out + +$MVN scala:doc -Dcheckstyle.skip=true -Denforcer.skip=true -Dspotless.skip=true -pl flink-scala 2> ${scaladoc_output} EXIT_CODE=$? if [ $EXIT_CODE != 0 ] ; then echo "ERROR in Scaladocs. Printing full output:" - cat scaladoc.out ; rm scaladoc.out + cat ${scaladoc_output} exit $EXIT_CODE fi diff --git a/tools/ci/shade.sh b/tools/ci/shade.sh index 70e99251280..51ea11e7540 100755 --- a/tools/ci/shade.sh +++ b/tools/ci/shade.sh @@ -17,10 +17,12 @@ # limitations under the License. ################################################################################ +jarContents=/tmp/allClasses + # Check the final fat jar for illegal or missing artifacts check_shaded_artifacts() { - jar tf build-target/lib/flink-dist*.jar > allClasses - ASM=`cat allClasses | grep '^org/objectweb/asm/' | wc -l` + jar tf build-target/lib/flink-dist*.jar > ${jarContents} + ASM=`cat ${jarContents} | grep '^org/objectweb/asm/' | wc -l` if [ "$ASM" != "0" ]; then echo "==============================================================================" echo "Detected '$ASM' unshaded asm dependencies in fat jar" @@ -28,7 +30,7 @@ check_shaded_artifacts() { return 1 fi - GUAVA=`cat allClasses | grep '^com/google/common' | wc -l` + GUAVA=`cat ${jarContents} | grep '^com/google/common' | wc -l` if [ "$GUAVA" != "0" ]; then echo "==============================================================================" echo "Detected '$GUAVA' guava dependencies in fat jar" @@ -36,7 +38,7 @@ check_shaded_artifacts() { return 1 fi - CODEHAUS_JACKSON=`cat allClasses | grep '^org/codehaus/jackson' | wc -l` + CODEHAUS_JACKSON=`cat ${jarContents} | grep '^org/codehaus/jackson' | wc -l` if [ "$CODEHAUS_JACKSON" != "0" ]; then echo "==============================================================================" echo "Detected '$CODEHAUS_JACKSON' unshaded org.codehaus.jackson classes in fat jar" @@ -44,7 +46,7 @@ check_shaded_artifacts() { return 1 fi - FASTERXML_JACKSON=`cat allClasses | grep '^com/fasterxml/jackson' | wc -l` + FASTERXML_JACKSON=`cat ${jarContents} | grep '^com/fasterxml/jackson' | wc -l` if [ "$FASTERXML_JACKSON" != "0" ]; then echo "==============================================================================" echo "Detected '$FASTERXML_JACKSON' unshaded com.fasterxml.jackson classes in fat jar" @@ -52,7 +54,7 @@ check_shaded_artifacts() { return 1 fi - SNAPPY=`cat allClasses | grep '^org/xerial/snappy' | wc -l` + SNAPPY=`cat ${jarContents} | grep '^org/xerial/snappy' | wc -l` if [ "$SNAPPY" = "0" ]; then echo "==============================================================================" echo "Missing snappy dependencies in fat jar" @@ -60,7 +62,7 @@ check_shaded_artifacts() { return 1 fi - IO_NETTY=`cat allClasses | grep '^io/netty' | wc -l` + IO_NETTY=`cat ${jarContents} | grep '^io/netty' | wc -l` if [ "$IO_NETTY" != "0" ]; then echo "==============================================================================" echo "Detected '$IO_NETTY' unshaded io.netty classes in fat jar" @@ -68,7 +70,7 @@ check_shaded_artifacts() { return 1 fi - ORG_NETTY=`cat allClasses | grep '^org/jboss/netty' | wc -l` + ORG_NETTY=`cat ${jarContents} | grep '^org/jboss/netty' | wc -l` if [ "$ORG_NETTY" != "0" ]; then echo "==============================================================================" echo "Detected '$ORG_NETTY' unshaded org.jboss.netty classes in fat jar" @@ -76,7 +78,7 @@ check_shaded_artifacts() { return 1 fi - ZOOKEEPER=`cat allClasses | grep '^org/apache/zookeeper' | wc -l` + ZOOKEEPER=`cat ${jarContents} | grep '^org/apache/zookeeper' | wc -l` if [ "$ZOOKEEPER" != "0" ]; then echo "==============================================================================" echo "Detected '$ZOOKEEPER' unshaded org.apache.zookeeper classes in fat jar" @@ -84,7 +86,7 @@ check_shaded_artifacts() { return 1 fi - CURATOR=`cat allClasses | grep '^org/apache/curator' | wc -l` + CURATOR=`cat ${jarContents} | grep '^org/apache/curator' | wc -l` if [ "$CURATOR" != "0" ]; then echo "==============================================================================" echo "Detected '$CURATOR' unshaded org.apache.curator classes in fat jar" @@ -92,7 +94,7 @@ check_shaded_artifacts() { return 1 fi - FLINK_PYTHON=`cat allClasses | grep '^org/apache/flink/python' | wc -l` + FLINK_PYTHON=`cat ${jarContents} | grep '^org/apache/flink/python' | wc -l` if [ "$FLINK_PYTHON" != "0" ]; then echo "==============================================================================" echo "Detected that the Flink Python artifact is in the dist jar" @@ -100,7 +102,7 @@ check_shaded_artifacts() { return 1 fi - HADOOP=`cat allClasses | grep '^org/apache/hadoop' | wc -l` + HADOOP=`cat ${jarContents} | grep '^org/apache/hadoop' | wc -l` if [ "$HADOOP" != "0" ]; then echo "==============================================================================" echo "Detected '$HADOOP' Hadoop classes in the dist jar" @@ -114,9 +116,9 @@ check_shaded_artifacts() { # Check the S3 fs implementations' fat jars for illegal or missing artifacts check_shaded_artifacts_s3_fs() { VARIANT=$1 - jar tf flink-filesystems/flink-s3-fs-${VARIANT}/target/flink-s3-fs-${VARIANT}*.jar > allClasses + jar tf flink-filesystems/flink-s3-fs-${VARIANT}/target/flink-s3-fs-${VARIANT}*.jar > ${jarContents} - if [ ! `cat allClasses | grep '^META-INF/services/org\.apache\.flink\.core\.fs\.FileSystemFactory$'` ]; then + if [ ! `cat ${jarContents} | grep '^META-INF/services/org\.apache\.flink\.core\.fs\.FileSystemFactory$'` ]; then echo "==============================================================================" echo "${VARIANT}: File does not exist: services/org.apache.flink.core.fs.FileSystemFactory" echo "==============================================================================" diff --git a/tools/ci/verify_bundled_optional.sh b/tools/ci/verify_bundled_optional.sh index 40e761ed3e6..a570ae92afb 100755 --- a/tools/ci/verify_bundled_optional.sh +++ b/tools/ci/verify_bundled_optional.sh @@ -26,7 +26,7 @@ source "${CI_DIR}/maven-utils.sh" cd "$FLINK_ROOT" || exit -dependency_plugin_output=${CI_DIR}/optional_dep.txt +dependency_plugin_output=/tmp/optional_dep.txt run_mvn dependency:tree -B > "${dependency_plugin_output}" diff --git a/tools/ci/verify_scala_suffixes.sh b/tools/ci/verify_scala_suffixes.sh index 2af7619a51a..22653d2dafb 100755 --- a/tools/ci/verify_scala_suffixes.sh +++ b/tools/ci/verify_scala_suffixes.sh @@ -48,7 +48,7 @@ source "${CI_DIR}/maven-utils.sh" cd "$FLINK_ROOT" || exit -dependency_plugin_output=${CI_DIR}/dep.txt +dependency_plugin_output=/tmp/dep.txt run_mvn dependency:tree -Dincludes=org.scala-lang,:*_2.1*:: ${MAVEN_ARGUMENTS} > "${dependency_plugin_output}" EXIT_CODE=$?