dak/integration-tests/dinstall

# -*- mode: sh -*-
#
# © 2019 Niels Thykier <niels@thykier.net>
# License: GPL-2+
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

. ${DAK_ROOT:?}/integration-tests/common

declare public_archives=(ftp-master debian-debug)
declare -r scriptdir=${DAKBASE}/scripts
declare -r overridedir=${scriptdir}/override # debian
echo "Dir::Override \"${overridedir}\";" >> "${DAK_CONFIG}"
declare -r pdiff_tempdir="${DAKBASE}/_dinstall-pdiff-temp"
declare -r upload_dir="${DAKBASE}/uploads"

DINSTALL_COUNTER=0

mkdir -p $upload_dir

setup_empty_archive() {
    section_start "setup_empty_archive[collapsed=true]" "Setup empty archive"

    dak admin archive add ftp-master "${DAKBASE}/ftp-master" "main archive"
    dak admin archive add debian-debug "${DAKBASE}/debian-debug" "debug archive"
    dak admin architecture add amd64 AMD64
    dak admin architecture add i386 "Intel ia32"
    HAVE_EXPERIMENTAL=
    HAVE_TPU=

    section_end setup_empty_archive
}

setup_unstable() {
    section_start "setup_debian_unstable[collapsed=true]" "Debian-like unstable"

    # unstable-debug + unstable
    dak admin suite add-all-arches unstable-debug "" codename=sid-debug archive=debian-debug \
        allowcsset=no byhash=yes accept_binary_uploads=False accept_source_uploads=False
    dak admin suite-component add unstable-debug main contrib non-free-firmware non-free


    dak admin suite add-all-arches unstable "" codename=sid archive=ftp-master \
        allowcsset=no byhash=yes overrideprocess=True debugsuite=unstable-debug \
        changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog
    dak admin suite-component add unstable main contrib non-free-firmware non-free
    dak admin suite add-build-queue unstable buildd-unstable buildd-sid build-queues

    section_end setup_debian_unstable
}

setup_debian_like_archive() {
    section_start "setup_debian_archive[collapsed=true]" "Debian-like Archive setup"
    setup_empty_archive
    setup_unstable

    # testing-debug + testing
    dak admin suite add-all-arches testing-debug "" codename=buster-debug archive=debian-debug \
        allowcsset=no byhash=yes accept_binary_uploads=False accept_source_uploads=False
    dak admin suite-component add testing-debug main contrib non-free-firmware non-free

    dak admin suite add-all-arches testing "" codename=buster archive=ftp-master \
        allowcsset=yes byhash=yes \
        changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog \
        changelog=dists/testing/ChangeLog \
        overrideprocess=True overrideorigin=unstable debugsuite=testing-debug
    dak admin suite-component add testing main contrib non-free-firmware non-free

    # testing-proposed-updates
    dak admin suite add-all-arches testing-proposed-updates "" \
        codename=buster-proposed-updates archive=ftp-master \
        allowcsset=yes byhash=yes \
        changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog \
        overridesuite=testing
    dak admin suite-component add testing-proposed-updates main contrib non-free-firmware non-free
    dak admin suite add-build-queue testing-proposed-updates buildd-testing-proposed-updates buildd-buster-proposed-updates build-queues
    HAVE_TPU=1

    
    # experimental
    dak admin suite add-all-arches experimental "" codename=rc-buggy archive=ftp-master \
        allowcsset=no byhash=yes \
        changelog_url=http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog \
        notautomatic=yes butautomaticupgrades=no \
        overridesuite=unstable overridecodename=sid
    dak admin suite-component add experimental main contrib non-free-firmware non-free
    dak admin suite add-build-queue experimental buildd-experimental buildd-rc-buggy build-queues
    HAVE_EXPERIMENTAL=1


    # Version constraints
    # Run "dak admin v-c list" to see them

    # experimental
    for suite in unstable testing ; do
        dak admin v-c add experimental MustBeNewerThan $suite
    done

    # testing-proposed-updates
    dak admin v-c add testing-proposed-updates Enhances testing
    dak admin v-c add testing-proposed-updates MustBeNewerThan testing
    # unstable
    dak admin v-c add unstable MustBeNewerThan testing

    # Lintian check
    cat > ${DAKBASE}/etc/lintian.tags <<-EOF
	lintian:
	  nonfatal:
	    - wrong-file-owner-uid-or-gid
	  fatal:
	    - binary-in-etc
	EOF

    echo "Dinstall::LintianTags \"${DAKBASE}/etc/lintian.tags\";" >> ${DAKBASE}/etc/dak.conf
    section_end setup_debian_archive
}

upload_changes() {
    dcmd cp -v "$@" $upload_dir
}

reupload_changes() {
    dak admin forget-signature "$@"
    dcmd cp -v "$@" $upload_dir
}

process_uploads() {
    dak process-upload -d $upload_dir --automatic
}

dinstall() {
    DINSTALL_COUNTER=$(( DINSTALL_COUNTER + 1 ))
    dinstall_name="dinstall_${DINSTALL_COUNTER}"

    section_start "${dinstall_name}[collapsed=true]" "dinstall #${DINSTALL_COUNTER}"

    echo "STARTING dinstall"
    dinstall_stage_start_up
    dinstall_stage_indices
    dinstall_stage_packages_contents
    dinstall_stage_dists

# state scripts (mostly OMITTED)
    # mkmaintainers
    # copyoverrides
    # mklslar
    # mkfilesindices
    # mkchecksums
    # dedup
    dak archive-dedup-pool
    # signotherfiles
    # mirror hardlinks
# UNLOCK UNCHECKED
# UNLOCK DAILY
    echo "FINISHED dinstall"
    section_end  "${dinstall_name}"
}

dinstall_stage_start_up() {
    # LOCK DAILY
    # LOCK UNCHECKED
    # run p-u-new (OMITTED)
    # run backports-policy (OMITTED)
    # cruft
    echo "Checking for cruft in overrides"
    dak check-overrides
}

dinstall_stage_indices() {
    # run dominate
  
    echo "Removing obsolete source and binary associations"
    dak dominate
    # NB: testing-debug is omitted in PROD as well (presumably
    # because the RT tooling handles that part)
    dak manage-debug-suites unstable-debug experimental-debug
    
    # run autocruft
    autocruft

    # run import-keyring (orig function fingerprints)
    # Skip: fingerprints / import-kerings
    # - It is used for sync'ing the deiban keyrings and emailing
    #   d-project.  Both of which we can do without in a test.

    # run overrides
    echo "Writing overrides into text files"
    mkdir -p $overridedir
    (cd $overridedir && dak make-overrides)

    # run pkg-file-mapping (orig function mpfm)
    mpfm
}

dinstall_stage_packages_contents() {
    # run apt-ftparchive (orig function packages)
    echo "  Generating Packages/Sources for: ${public_archives[*]}"
    dak generate-packages-sources2 -a "${public_archives[*]}"
    echo "  Generating Contents for: ${public_archives[*]}"
    dak contents generate -a "${public_archives[*]}"
}

dinstall_stage_dists() {
    # run pdiff
    echo "Generating pdiff files"
    mkdir -p "${pdiff_tempdir}"
    dak generate-index-diffs -d "${pdiff_tempdir}" -a "${public_archives[*]}"
    # run release files
    echo "Generating Release files"
    dak generate-releases -a "${public_archives[*]}"
    # run cleanup (OMITTED) - it removes tempfiles that we do not have
}

autocruft() {
    echo "Check for obsolete binary packages"
    dak auto-decruft -s unstable
    if [ -n "$HAVE_EXPERIMENTAL" ]
    then
        dak auto-decruft -s experimental --if-newer-version-in unstable --if-newer-version-in-rm-msg "NVIU"
    fi
    if [ -n "$HAVE_TPU" ]
    then
        dak auto-decruft -s testing-proposed-updates --if-newer-version-in testing --if-newer-version-in-rm-msg "NVIT" --decruft-equal-versions
    fi
}

get_archiveroot() {
    local archivename="$1"
    local query="SELECT path FROM archive WHERE name='${archivename}'"
    local archiveroot="$(psql -tAc "${query}")"
    if [[ -z ${archiveroot} ]]; then
        echo "get_archiveroot: couldn't get archiveroot for '${archivename}'" >&2
        return 1
    fi
    echo "${archiveroot}"
}

mpfm() {
    local archiveroot

    echo "Generating package / file mapping"
    for archive in "${public_archives[@]}"; do
        echo "  archive: ${archive}"
        archiveroot="$(get_archiveroot "${archive}")"
        mkdir -p "${archiveroot}/indices"
        dak make-pkg-file-mapping "${archive}" | bzip2 -9 > "${archiveroot}/indices/package-file.map.bz2"
    done
}

# process NEW policy queue
function do_new () {
    echo "Doing NEW processing"
    dak process-policy new

    echo "Cleanup NEW/Backports NEW"
    dak clean-suites -a new
}