From 742685b76c7f001a08799a539cad2bb683d5d29d Mon Sep 17 00:00:00 2001 From: Matthias Pohl Date: Mon, 6 Feb 2023 19:55:57 +0100 Subject: [PATCH] [FLINK-30921][ci] Adds mirrors instead of relying on a single source for Ubuntu packages We don't want to rely on an external service for retrieving mirrors dynamically to prevent supply chain attacks. Therefore, we selected a few mirrors manually. This prevents us from relying on a dynamic mirror list based on the CI machines location which might be a performance issue. But generally, we want to rely on the Azure mirror anyway. The fallback list is only meant to jump in if Azure's mirror is not accessible temporarily. Signed-off-by: Matthias Pohl --- tools/azure-pipelines/e2e-template.yml | 9 ++++++ tools/ci/ubuntu-mirror-list.txt | 40 ++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 tools/ci/ubuntu-mirror-list.txt diff --git a/tools/azure-pipelines/e2e-template.yml b/tools/azure-pipelines/e2e-template.yml index c48c402d507..11d8dd77e96 100644 --- a/tools/azure-pipelines/e2e-template.yml +++ b/tools/azure-pipelines/e2e-template.yml @@ -100,6 +100,15 @@ jobs: source ./tools/ci/maven-utils.sh setup_maven + # the APT mirrors access is based on a proposal from https://github.com/actions/runner-images/issues/7048#issuecomment-1419426054 + echo "Configure APT mirrors" + mirror_file_path="/etc/apt/mirrors.txt" + default_ubuntu_mirror_url="http://azure.archive.ubuntu.com/ubuntu/" + + sudo cp ./tools/ci/ubuntu-mirror-list.txt ${mirror_file_path} + sudo sed -i "s~${default_ubuntu_mirror_url}~mirror+file:${mirror_file_path}~" /etc/apt/sources.list + sudo apt-get update + echo "Installing required software" sudo apt-get install -y bc libapr1 # install libssl1.0.0 for netty tcnative diff --git a/tools/ci/ubuntu-mirror-list.txt b/tools/ci/ubuntu-mirror-list.txt new file mode 100644 index 00000000000..5f7fa5b74bf --- /dev/null +++ b/tools/ci/ubuntu-mirror-list.txt @@ -0,0 +1,40 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# This mirror list can be used in CI to set up the Docker containers without only +# relying on the default Ubuntu mirror provided by Azure. +# FYI: apt ignores lines that are empty or start with '#' +# see https://manpages.ubuntu.com/manpages/jammy/en/man1/apt-transport-mirror.1.html for further documentation + +# primary mirror +http://azure.archive.ubuntu.com/ubuntu/ priority:1 + +# fallback mirrors selected from http://mirrors.ubuntu.com/ based on the assumption that CI +# is running on AliCloud machines or Azure machines located in the US + +# US-located mirrors +http://mirrors.ocf.berkeley.edu/ubuntu/ +http://mirror.math.ucdavis.edu/ubuntu/ +http://mirrors.mit.edu/ubuntu/ +https://mirror.ubuntu.serverforge.org/ + +# AliCloud mirror +http://mirrors.aliyun.com/ubuntu/ + +# general +http://archive.ubuntu.com/ubuntu/