diff --git a/scripts/gitlab-ce.sh b/scripts/gitlab-ce.sh index 4b09f44..bf5756a 100755 --- a/scripts/gitlab-ce.sh +++ b/scripts/gitlab-ce.sh @@ -1,14 +1,19 @@ #!/bin/bash set -e +_here=`dirname $(realpath $0)` +. ${_here}/helpers/apt-download + +[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1) + BASE_PATH="${TUNASYNC_WORKING_DIR}" - -APT_PATH="${BASE_PATH}/apt" YUM_PATH="${BASE_PATH}/yum" UBUNTU_VERSIONS=("trusty" "wily") DEBIAN_VERSIONS=("wheezy" "jessie" "stretch") +UBUNTU_PATH="${BASE_PATH}/ubuntu/" +DEBIAN_PATH="${BASE_PATH}/debian/" mkdir -p $UBUNTU_PATH $DEBIAN_PATH $YUM_PATH @@ -35,44 +40,23 @@ sslverify=0 EOF reposync -c $cfg -d -p ${YUM_PATH} -createrepo -o ${YUM_PATH}/el6 ${YUM_PATH}/el6 -createrepo -o ${YUM_PATH}/el7 ${YUM_PATH}/el7 +createrepo --update -o ${YUM_PATH}/el6 ${YUM_PATH}/el6 +createrepo --update -o ${YUM_PATH}/el7 ${YUM_PATH}/el7 rm $cfg - -cfg="/tmp/gitlab-ce-ubuntu.list" -cat << EOF > ${cfg} -set mirror_path ${APT_PATH} -set nthreds 5 -set _tilde 0 - -EOF +base_url="https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu" for version in ${UBUNTU_VERSIONS[@]}; do - echo "deb https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu/ $version main" >> $cfg - echo "deb-i386 https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu/ $version main" >> $cfg - echo "deb-amd64 https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu/ $version main" >> $cfg - echo "deb-src https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu/ $version main" >> $cfg + apt-download-binary ${base_url} "$version" "main" "amd64" "${UBUNTU_PATH}" + apt-download-binary ${base_url} "$version" "main" "i386" "${UBUNTU_PATH}" done +echo "Ubuntu finished" -apt-mirror $cfg -rm $cfg - - -cfg="/tmp/gitlab-ce-debian.list" -cat << EOF > ${cfg} -set mirror_path ${APT_PATH} -set nthreds 5 -set _tilde 0 - -EOF +base_url="https://packages.gitlab.com/gitlab/gitlab-ce/debian" for version in ${DEBIAN_VERSIONS[@]}; do - echo "deb https://packages.gitlab.com/gitlab/gitlab-ce/debian/ $version main" >> $cfg - echo "deb-i386 https://packages.gitlab.com/gitlab/gitlab-ce/debian/ $version main" >> $cfg - echo "deb-amd64 https://packages.gitlab.com/gitlab/gitlab-ce/debian/ $version main" >> $cfg - echo "deb-src https://packages.gitlab.com/gitlab/gitlab-ce/debian/ $version main" >> $cfg + apt-download-binary ${base_url} "$version" "main" "amd64" "${DEBIAN_PATH}" + apt-download-binary ${base_url} "$version" "main" "i386" "${DEBIAN_PATH}" done +echo "Debian finished" -apt-mirror $cfg -rm $cfg # vim: ts=4 sts=4 sw=4 diff --git a/scripts/helpers/apt-download b/scripts/helpers/apt-download new file mode 100644 index 0000000..3f81a84 --- /dev/null +++ b/scripts/helpers/apt-download @@ -0,0 +1,78 @@ +#!/bin/bash +set -e +LOADED_APT_DOWNLOAD="yes" + +function check-and-download() { + remote_file=$1 + local_file=$2 + wget -q --spider ${remote_file} + if [ $? -eq 0 ]; then + echo "downloading ${remote_file}" + wget -q -N -O ${local_file} ${remote_file} + return + fi + return 0 +} + + +function apt-download-binary() { + base_url=$1 + dist=$2 + repo=$3 + arch=$4 + dest_base_dir=$5 + + dest_dir="${dest_base_dir}/dists/${dist}" + [ ! -d "$dest_dir" ] && mkdir -p "$dest_dir" + check-and-download "${base_url}/dists/${dist}/Contents-${arch}.gz" "${dest_dir}/Contents-${arch}.gz" + check-and-download "${base_url}/dists/${dist}/InRelease" "${dest_dir}/InRelease" || true + check-and-download "${base_url}/dists/${dist}/Release" "${dest_dir}/Release" + check-and-download "${base_url}/dists/${dist}/Release.gpg" "${dest_dir}/Release.gpg" || true + + dest_dir="${dest_base_dir}/dists/${dist}/${repo}/binary-${arch}" + [ ! -d "$dest_dir" ] && mkdir -p "$dest_dir" + + pkgidxes=('Packages' 'Packages.gz' 'Packages.bz2' 'Packages.xz') + pkgidx_content="" + for pkgidx in ${pkgidxes[@]}; do + pkgidx_file="${dest_dir}/${pkgidx}" + pkglist_url="${base_url}/dists/${dist}/${repo}/binary-${arch}/${pkgidx}" + check-and-download "${pkglist_url}" ${pkgidx_file} || true + if [ -z "${pkgidx_content}" -a -f ${pkgidx_file} ]; then + echo "getting packages index content" + case $pkgidx in + "*.bz2") + pkgidx_content=`bunzip2 -c ${pkgidx_file}` + ;; + "*.gz") + pkgidx_content=`gunzip -c ${pkgidx_file}` + ;; + *) + pkgidx_content=`cat ${pkgidx_file}` + ;; + esac + fi + done + + if [ -z "${pkgidx_content}" ]; then + echo "index is empty, failed" + return 1 + fi + + (echo -e "${pkgidx_content}" | grep '^Filename' | cut -d' ' -f 2) | while read pkg_filename; do + dest_filename="${dest_base_dir}/${pkg_filename}" + dest_dir=`dirname ${dest_filename}` + [ ! -d "$dest_dir" ] && mkdir -p "$dest_dir" + pkg_url="${base_url}/${pkg_filename}" + + if [ -f ${dest_filename} ]; then + echo "Skipping ${pkg_filename}" + else + echo "downloading ${pkg_url}" + # touch ${dest_filename} + wget -q -O ${dest_filename} ${pkg_url} || true + fi + + done + +}