update-vendor.sh

#!/usr/bin/env bash

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"

# Explicitly opt into go modules, even though we're inside a GOPATH directory
export GO111MODULE=on
# Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
export GOFLAGS=
# Ensure sort order doesn't depend on locale
export LANG=C
export LC_ALL=C
# Detect problematic GOPROXY settings that prevent lookup of dependencies
if [[ "${GOPROXY:-}" == "off" ]]; then
  kube::log::error "Cannot run hack/update-vendor.sh with \$GOPROXY=off"
  exit 1
fi

kube::golang::verify_go_version
kube::util::require-jq

TMP_DIR="${TMP_DIR:-$(mktemp -d /tmp/update-vendor.XXXX)}"
LOG_FILE="${LOG_FILE:-${TMP_DIR}/update-vendor.log}"
kube::log::status "logfile at ${LOG_FILE}"

if [ -z "${BASH_XTRACEFD:-}" ]; then
  exec 19> "${LOG_FILE}"
  export BASH_XTRACEFD="19"
  set -x
fi

# ensure_require_replace_directives_for_all_dependencies:
# - ensures all existing 'require' directives have an associated 'replace' directive pinning a version
# - adds explicit 'require' directives for all transitive dependencies
# - adds explicit 'replace' directives for all require directives (existing 'replace' directives take precedence)
function ensure_require_replace_directives_for_all_dependencies() {
  local local_tmp_dir
  local_tmp_dir=$(mktemp -d "${TMP_DIR}/pin_replace.XXXX")

  # collect 'require' directives that actually specify a version
  local require_filter='(.Version != null) and (.Version != "v0.0.0") and (.Version != "v0.0.0-00010101000000-000000000000")'
  # collect 'replace' directives that unconditionally pin versions (old=new@version)
  local replace_filter='(.Old.Version == null) and (.New.Version != null)'

  # Capture local require/replace directives before running any go commands that can modify the go.mod file
  local require_json="${local_tmp_dir}/require.json"
  local replace_json="${local_tmp_dir}/replace.json"
  go mod edit -json | jq -r ".Require // [] | sort | .[] | select(${require_filter})" > "${require_json}"
  go mod edit -json | jq -r ".Replace // [] | sort | .[] | select(${replace_filter})" > "${replace_json}"

  # 1a. Ensure replace directives have an explicit require directive
  jq -r '"-require \(.Old.Path)@\(.New.Version)"' < "${replace_json}" | xargs -L 100 go mod edit -fmt
  # 1b. Ensure require directives have a corresponding replace directive pinning a version
  jq -r '"-replace \(.Path)=\(.Path)@\(.Version)"' < "${require_json}" | xargs -L 100 go mod edit -fmt
  jq -r '"-replace \(.Old.Path)=\(.New.Path)@\(.New.Version)"' < "${replace_json}" | xargs -L 100 go mod edit -fmt

  # 2. Propagate root replace/require directives into staging modules, in case we are downgrading, so they don't bump the root required version back up
  for repo in $(kube::util::list_staging_repos); do
    pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1
      jq -r '"-require \(.Path)@\(.Version)"' < "${require_json}" | xargs -L 100 go mod edit -fmt
      jq -r '"-replace \(.Path)=\(.Path)@\(.Version)"' < "${require_json}" | xargs -L 100 go mod edit -fmt
      jq -r '"-replace \(.Old.Path)=\(.New.Path)@\(.New.Version)"' < "${replace_json}" | xargs -L 100 go mod edit -fmt
    popd >/dev/null 2>&1
  done

  # 3. Add explicit require directives for indirect dependencies
  go list -m -json all | jq -r 'select(.Main != true) | select(.Indirect == true) | "-require \(.Path)@\(.Version)"'          | xargs -L 100 go mod edit -fmt

  # 4. Add explicit replace directives pinning dependencies that aren't pinned yet
  go list -m -json all | jq -r 'select(.Main != true) | select(.Replace == null)  | "-replace \(.Path)=\(.Path)@\(.Version)"' | xargs -L 100 go mod edit -fmt
}

function group_replace_directives() {
  local local_tmp_dir
  local_tmp_dir=$(mktemp -d "${TMP_DIR}/group_replace.XXXX")
  local go_mod_replace="${local_tmp_dir}/go.mod.replace.tmp"
  local go_mod_noreplace="${local_tmp_dir}/go.mod.noreplace.tmp"
  # separate replace and non-replace directives
  awk "
     # print lines between 'replace (' ... ')' lines
     /^replace [(]/      { inreplace=1; next                   }
     inreplace && /^[)]/ { inreplace=0; next                   }
     inreplace           { print > \"${go_mod_replace}\"; next }
     
     # print ungrouped replace directives with the replace directive trimmed
     /^replace [^(]/ { sub(/^replace /,\"\"); print > \"${go_mod_replace}\"; next }
     
     # otherwise print to the noreplace file
     { print > \"${go_mod_noreplace}\" }
  " < go.mod
  {
    cat "${go_mod_noreplace}";
    echo "replace (";
    cat "${go_mod_replace}";
    echo ")";
  } > go.mod

  go mod edit -fmt
}

function add_generated_comments() {
  local local_tmp_dir
  local_tmp_dir=$(mktemp -d "${TMP_DIR}/add_generated_comments.XXXX")
  local go_mod_nocomments="${local_tmp_dir}/go.mod.nocomments.tmp"

  # drop comments before the module directive
  awk "
     BEGIN           { dropcomments=1 }
     /^module /      { dropcomments=0 }
     dropcomments && /^\/\// { next }
     { print }
  " < go.mod > "${go_mod_nocomments}"

  # Add the specified comments
  local comments="${1}"
  {
    echo "${comments}"
    echo ""
    cat "${go_mod_nocomments}"
   } > go.mod
  
  # Format
  go mod edit -fmt
}


# Phase 1: ensure go.mod files for staging modules and main module

for repo in $(kube::util::list_staging_repos); do
  pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1
    if [[ ! -f go.mod ]]; then
      kube::log::status "go.mod: initialize ${repo}"
      rm -f Godeps/Godeps.json # remove before initializing, staging Godeps are not authoritative
      go mod init "k8s.io/${repo}"
      go mod edit -fmt
    fi
  popd >/dev/null 2>&1
done

if [[ ! -f go.mod ]]; then
  kube::log::status "go.mod: initialize k8s.io/kubernetes"
  go mod init "k8s.io/kubernetes"
  rm -f Godeps/Godeps.json # remove after initializing
fi


# Phase 2: ensure staging repo require/replace directives

kube::log::status "go.mod: update staging references"
# Prune
go mod edit -json | jq -r '.Require[]? | select(.Version == "v0.0.0")                 | "-droprequire \(.Path)"'     | xargs -L 100 go mod edit -fmt
go mod edit -json | jq -r '.Replace[]? | select(.New.Path | startswith("./staging/")) | "-dropreplace \(.Old.Path)"' | xargs -L 100 go mod edit -fmt
# Readd
kube::util::list_staging_repos | xargs -n 1 -I {} echo "-require k8s.io/{}@v0.0.0"                  | xargs -L 100 go mod edit -fmt
kube::util::list_staging_repos | xargs -n 1 -I {} echo "-replace k8s.io/{}=./staging/src/k8s.io/{}" | xargs -L 100 go mod edit -fmt


# Phase 3: capture required (minimum) versions from all modules, and replaced (pinned) versions from the root module

# pin referenced versions
ensure_require_replace_directives_for_all_dependencies
# resolves/expands references in the root go.mod (if needed)
go mod tidy >>"${LOG_FILE}" 2>&1
# pin expanded versions
ensure_require_replace_directives_for_all_dependencies
# group replace directives
group_replace_directives

# Phase 4: copy root go.mod to staging dirs and rewrite

kube::log::status "go.mod: propagate to staging modules"
for repo in $(kube::util::list_staging_repos); do
  pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1
    echo "=== propagating to ${repo}" >> "${LOG_FILE}"
    # copy root go.mod, changing module name
    sed "s#module k8s.io/kubernetes#module k8s.io/${repo}#" < "${KUBE_ROOT}/go.mod" > "${KUBE_ROOT}/staging/src/k8s.io/${repo}/go.mod"
    # remove `require` directives for staging components (will get re-added as needed by `go list`)
    kube::util::list_staging_repos | xargs -n 1 -I {} echo "-droprequire k8s.io/{}"   | xargs -L 100 go mod edit
    # rewrite `replace` directives for staging components to point to peer directories
    kube::util::list_staging_repos | xargs -n 1 -I {} echo "-replace k8s.io/{}=../{}" | xargs -L 100 go mod edit
  popd >/dev/null 2>&1
done


# Phase 5: sort and tidy staging components

kube::log::status "go.mod: sorting staging modules"
# tidy staging repos in reverse dependency order.
# the content of dependencies' go.mod files affects what `go mod tidy` chooses to record in a go.mod file.
tidy_unordered="${TMP_DIR}/tidy_unordered.txt"
kube::util::list_staging_repos | xargs -I {} echo "k8s.io/{}" > "${tidy_unordered}"
rm -f "${TMP_DIR}/tidy_deps.txt"
# SC2094 checks that you do not read and write to the same file in a pipeline.
# We do read from ${tidy_unordered} in the pipeline and mention it within the
# pipeline (but only ready it again) so we disable the lint to assure shellcheck
# that :this-is-fine:
# shellcheck disable=SC2094
while IFS= read -r repo; do
  # record existence of the repo to ensure modules with no peer relationships still get included in the order
  echo "${repo} ${repo}" >> "${TMP_DIR}/tidy_deps.txt"

  pushd "${KUBE_ROOT}/staging/src/${repo}" >/dev/null 2>&1
    # save the original go.mod, since go list doesn't just add missing entries, it also removes specific required versions from it
    tmp_go_mod="${TMP_DIR}/tidy_${repo/\//_}_go.mod.original"
    tmp_go_deps="${TMP_DIR}/tidy_${repo/\//_}_deps.txt"
    cp go.mod "${tmp_go_mod}"

    {
      echo "=== sorting ${repo}"
      # 'go list' calculates direct imports and updates go.mod so that go list -m lists our module dependencies
      echo "=== computing imports for ${repo}"
      go list all
      echo "=== computing tools imports for ${repo}"
      go list -tags=tools all
    } >> "${LOG_FILE}" 2>&1

    # capture module dependencies 
    go list -m -f '{{if not .Main}}{{.Path}}{{end}}' all > "${tmp_go_deps}"

    # restore the original go.mod file
    cp "${tmp_go_mod}" go.mod

    # list all module dependencies
    for dep in $(join "${tidy_unordered}" "${tmp_go_deps}"); do
      # record the relationship (put dep first, because we want to sort leaves first)
      echo "${dep} ${repo}" >> "${TMP_DIR}/tidy_deps.txt"
      # switch the required version to an explicit v0.0.0 (rather than an unknown v0.0.0-00010101000000-000000000000)
      go mod edit -require "${dep}@v0.0.0"
    done
  popd >/dev/null 2>&1
done < "${tidy_unordered}"

kube::log::status "go.mod: tidying"
for repo in $(tsort "${TMP_DIR}/tidy_deps.txt"); do
  pushd "${KUBE_ROOT}/staging/src/${repo}" >/dev/null 2>&1
    echo "=== tidying ${repo}" >> "${LOG_FILE}"

    # prune replace directives that pin to the naturally selected version.
    # do this before tidying, since tidy removes unused modules that
    # don't provide any relevant packages, which forgets which version of the 
    # unused transitive dependency we had a require directive for,
    # and prevents pruning the matching replace directive after tidying.
    go list -m -json all |
      jq -r 'select(.Replace != null) | 
             select(.Path == .Replace.Path) | 
             select(.Version == .Replace.Version) | 
             "-dropreplace \(.Replace.Path)"' |
    xargs -L 100 go mod edit -fmt

    go mod tidy -v >>"${LOG_FILE}" 2>&1

    # disallow transitive dependencies on k8s.io/kubernetes
    loopback_deps=()
    kube::util::read-array loopback_deps < <(go list all 2>/dev/null | grep k8s.io/kubernetes/ || true)
    if [[ -n ${loopback_deps[*]:+"${loopback_deps[*]}"} ]]; then
      kube::log::error "Disallowed ${repo} -> k8s.io/kubernetes dependencies exist via the following imports:
$(go mod why "${loopback_deps[@]}")"
      exit 1
    fi

    # prune unused pinned replace directives
    comm -23 \
      <(go mod edit -json | jq -r '.Replace[] | .Old.Path' | sort) \
      <(go list -m -json all | jq -r .Path | sort) |
    xargs -L 1 -I {} echo "-dropreplace={}" |
    xargs -L 100 go mod edit -fmt

    # prune replace directives that pin to the naturally selected version
    go list -m -json all |
      jq -r 'select(.Replace != null) | 
             select(.Path == .Replace.Path) | 
             select(.Version == .Replace.Version) | 
             "-dropreplace \(.Replace.Path)"' |
    xargs -L 100 go mod edit -fmt

  popd >/dev/null 2>&1
done
echo "=== tidying root" >> "${LOG_FILE}"
go mod tidy >>"${LOG_FILE}" 2>&1


# Phase 6: add generated comments to go.mod files
kube::log::status "go.mod: adding generated comments"
add_generated_comments "
// This is a generated file. Do not edit directly.
// Run hack/pin-dependency.sh to change pinned dependency versions.
// Run hack/update-vendor.sh to update go.mod files and the vendor directory.
"
for repo in $(kube::util::list_staging_repos); do
  pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1
    add_generated_comments "// This is a generated file. Do not edit directly."
  popd >/dev/null 2>&1
done


# Phase 6: rebuild vendor directory

kube::log::status "vendor: running 'go mod vendor'"
go mod vendor >>"${LOG_FILE}" 2>&1

# sort recorded packages for a given vendored dependency in modules.txt.
# `go mod vendor` outputs in imported order, which means slight go changes (or different platforms) can result in a differently ordered modules.txt.
# scan                 | prefix comment lines with the module name       | sort field 1  | strip leading text on comment lines
awk '{if($1=="#") print $2 " " $0; else print}' < vendor/modules.txt | sort -k1,1 -s | sed 's/.*#/#/' > "${TMP_DIR}/modules.txt.tmp"
mv "${TMP_DIR}/modules.txt.tmp" vendor/modules.txt

# create a symlink in vendor directory pointing to the staging components.
# This lets other packages and tools use the local staging components as if they were vendored.
for repo in $(kube::util::list_staging_repos); do
  rm -fr "${KUBE_ROOT}/vendor/k8s.io/${repo}"
  ln -s "../../staging/src/k8s.io/${repo}" "${KUBE_ROOT}/vendor/k8s.io/${repo}"
done

kube::log::status "vendor: updating BUILD files"
# Assume that anything imported through vendor doesn't need Bazel to build.
# Prune out any Bazel build files, since these can break the build due to
# missing dependencies that aren't included by go mod vendor.
find vendor/ -type f \( -name BUILD -o -name BUILD.bazel -o -name WORKSPACE \) -exec rm -f {} \;
hack/update-bazel.sh >>"${LOG_FILE}" 2>&1

kube::log::status "vendor: updating LICENSES file"
hack/update-vendor-licenses.sh >>"${LOG_FILE}" 2>&1

kube::log::status "vendor: creating OWNERS file"
rm -f "Godeps/OWNERS" "vendor/OWNERS"
cat <<__EOF__ > "Godeps/OWNERS"
# See the OWNERS docs at https://go.k8s.io/owners

approvers:
- dep-approvers
__EOF__
cp "Godeps/OWNERS" "vendor/OWNERS"