flink/tools/travis_controller.sh

#!/usr/bin/env bash
################################################################################
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

echo $M2_HOME
echo $PATH
echo $MAVEN_OPTS

mvn -version

echo "Commit: $(git rev-parse HEAD)"

CACHE_DIR="$HOME/flink_cache"
CACHE_BUILD_DIR="$CACHE_DIR/$TRAVIS_BUILD_NUMBER"
CACHE_FLINK_DIR="$CACHE_BUILD_DIR/flink"

echo "Flink cache location: ${CACHE_FLINK_DIR}"

HERE="`dirname \"$0\"`"				# relative
HERE="`( cd \"$HERE\" && pwd )`" 	# absolutized and normalized
if [ -z "$HERE" ] ; then
	# error; for some reason, the path is not accessible
	# to the script (e.g. permissions re-evaled after suid)
	exit 1  # fail
fi

source "${HERE}/travis/fold.sh"
source "${HERE}/travis/stage.sh"
source "${HERE}/travis/shade.sh"

print_system_info() {
	FOLD_ESCAPE="\x0d\x1b"
	COLOR_ON="\x5b\x30\x4b\x1b\x5b\x33\x33\x3b\x31\x6d"
	COLOR_OFF="\x1b\x5b\x30\x6d"

	start_fold "cpu_info" "CPU information"
	lscpu
	end_fold "cpu_info"

	start_fold "mem_info" "Memory information"
	cat /proc/meminfo
	end_fold "mem_info"

	start_fold "disk_info" "Disk information"
	df -hH
	end_fold "disk_info"

	start_fold "cache_info" "Cache information"
	echo "Maven: $(du -s --si $HOME/.m2)"
	echo "Flink: $(du -s --si $HOME/flink_cache)"
	echo "Maven (binaries): $(du -s --si $HOME/maven_cache)"
	echo "gems: $(du -s --si $HOME/gem_cache)"
	end_fold "cache_info"
}

print_system_info

function deleteOldCaches() {
	while read CACHE_DIR; do
		local old_number="${CACHE_DIR##*/}"
		if [ "$old_number" -lt "$TRAVIS_BUILD_NUMBER" ]; then
			echo "Deleting old cache $CACHE_DIR"
			rm -rf "$CACHE_DIR"
		fi
	done
}

# delete leftover caches from previous builds; except the most recent
find "$CACHE_DIR" -mindepth 1 -maxdepth 1 | grep -v "$TRAVIS_BUILD_NUMBER" | sort -Vr | tail -n +2 | deleteOldCaches

STAGE=$1
echo "Current stage: \"$STAGE\""

EXIT_CODE=0

# Run actual compile&test steps
if [ $STAGE == "$STAGE_COMPILE" ]; then
	MVN="mvn clean install -nsu -Dflink.convergence.phase=install -Pcheck-convergence -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.javadoc.skip=true -B -DskipTests $PROFILE -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Dmaven.wagon.http.pool=false"
	$MVN
	EXIT_CODE=$?

    if [ $EXIT_CODE == 0 ]; then
        printf "\n\n==============================================================================\n"
        printf "Checking scala suffixes\n"
        printf "==============================================================================\n"

        ./tools/verify_scala_suffixes.sh "${PROFILE}"
        EXIT_CODE=$?
    else
        printf "\n==============================================================================\n"
        printf "Previous build failure detected, skipping scala-suffixes check.\n"
        printf "==============================================================================\n"
    fi
    
    if [ $EXIT_CODE == 0 ]; then
        check_shaded_artifacts
        EXIT_CODE=$(($EXIT_CODE+$?))
        check_shaded_artifacts_s3_fs hadoop
        EXIT_CODE=$(($EXIT_CODE+$?))
        check_shaded_artifacts_s3_fs presto
        EXIT_CODE=$(($EXIT_CODE+$?))
        check_shaded_artifacts_connector_elasticsearch 2
        EXIT_CODE=$(($EXIT_CODE+$?))
        check_shaded_artifacts_connector_elasticsearch 5
        EXIT_CODE=$(($EXIT_CODE+$?))
        check_shaded_artifacts_connector_elasticsearch 6
        EXIT_CODE=$(($EXIT_CODE+$?))
    else
        echo "=============================================================================="
        echo "Previous build failure detected, skipping shaded dependency check."
        echo "=============================================================================="
    fi

    if [ $EXIT_CODE == 0 ]; then
        echo "Creating cache build directory $CACHE_FLINK_DIR"
        mkdir -p "$CACHE_FLINK_DIR"
    
        cp -r . "$CACHE_FLINK_DIR"

        function minimizeCachedFiles() {
            # reduces the size of the cached directory to speed up
            # the packing&upload / download&unpacking process
            # by removing files not required for subsequent stages
    
            # jars are re-built in subsequent stages, so no need to cache them (cannot be avoided)
            find "$CACHE_FLINK_DIR" -maxdepth 8 -type f -name '*.jar' \
            ! -path "$CACHE_FLINK_DIR/flink-formats/flink-csv/target/flink-csv*.jar" \
            ! -path "$CACHE_FLINK_DIR/flink-formats/flink-json/target/flink-json*.jar" \
            ! -path "$CACHE_FLINK_DIR/flink-formats/flink-avro/target/flink-avro*.jar" \
            ! -path "$CACHE_FLINK_DIR/flink-runtime/target/flink-runtime*tests.jar" \
            ! -path "$CACHE_FLINK_DIR/flink-streaming-java/target/flink-streaming-java*tests.jar" \
            ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-dist*.jar" \
            ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-table_*.jar" \
            ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-table-blink*.jar" \
            ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/opt/flink-python*.jar" \
            ! -path "$CACHE_FLINK_DIR/flink-connectors/flink-connector-elasticsearch-base/target/flink-*.jar" \
            ! -path "$CACHE_FLINK_DIR/flink-connectors/flink-connector-kafka-base/target/flink-*.jar" \
            ! -path "$CACHE_FLINK_DIR/flink-table/flink-table-planner/target/flink-table-planner*tests.jar" | xargs rm -rf
    
            # .git directory
            # not deleting this can cause build stability issues
            # merging the cached version sometimes fails
            rm -rf "$CACHE_FLINK_DIR/.git"
        }
    
        start_fold "minimize_cache" "Minimizing cache"
        travis_time_start
        minimizeCachedFiles
        travis_time_finish
        end_fold "minimize_cache"
    else
        echo "=============================================================================="
        echo "Previous build failure detected, skipping cache setup."
        echo "=============================================================================="
    fi
elif [ $STAGE != "$STAGE_CLEANUP" ]; then
	if ! [ -e $CACHE_FLINK_DIR ]; then
		echo "Cached flink dir $CACHE_FLINK_DIR does not exist. Exiting build."
		exit 1
	fi
	# merged compiled flink into local clone
	# this prevents the cache from being re-uploaded
	start_fold "merge_cache" "Merging cache"
	travis_time_start
	cp -RT "$CACHE_FLINK_DIR" "."
	travis_time_finish
	end_fold "merge_cache"

	start_fold "adjust_timestamps" "Adjusting timestamps"
	travis_time_start
	# adjust timestamps to prevent recompilation
	find . -type f -name '*.java' | xargs touch
	find . -type f -name '*.scala' | xargs touch
	# wait a bit for better odds of different timestamps
	sleep 5
	find . -type f -name '*.class' | xargs touch
	find . -type f -name '*.timestamp' | xargs touch
	travis_time_finish
	end_fold "adjust_timestamps"

	TEST="$STAGE" "./tools/travis_watchdog.sh" 300
	EXIT_CODE=$?
elif [ $STAGE == "$STAGE_CLEANUP" ]; then
	echo "Cleaning up $CACHE_BUILD_DIR"
	rm -rf "$CACHE_BUILD_DIR"
else
    echo "Invalid Stage specified: $STAGE"
    exit 1
fi

# Exit code for Travis build success/failure
exit $EXIT_CODE