Skip to content

Commit a1e6946

Browse files
hugodutkahugodutka
committed
enable go test cache in test-go-pg
1 parent 4ba317c commit a1e6946

File tree

4 files changed

+163
-38
lines changed

4 files changed

+163
-38
lines changed

‎.github/actions/setup-go-paths/action.yml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: "Setup Go Paths"
2+
description: Overrides Go paths like GOCACHE and GOMODCACHE to use temporary directories.
3+
outputs:
4+
gocache:
5+
description: "Value of GOCACHE"
6+
value: ${{ steps.paths.outputs.gocache }}
7+
gomodcache:
8+
description: "Value of GOMODCACHE"
9+
value: ${{ steps.paths.outputs.gomodcache }}
10+
gopath:
11+
description: "Value of GOPATH"
12+
value: ${{ steps.paths.outputs.gopath }}
13+
gotmp:
14+
description: "Value of GOTMPDIR"
15+
value: ${{ steps.paths.outputs.gotmp }}
16+
cached-dirs:
17+
description: "Go directories that should be cached between CI runs"
18+
value: ${{ steps.paths.outputs.cached-dirs }}
19+
runs:
20+
using: "composite"
21+
steps:
22+
- name: Override Go paths
23+
id: paths
24+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
25+
with:
26+
script: |
27+
const path = require('path');
28+
29+
// RUNNER_TEMP should be backed by a RAM disk on Windows if
30+
// coder/setup-ramdisk-action was used
31+
const runnerTemp = process.env.RUNNER_TEMP;
32+
const gocacheDir = path.join(runnerTemp, 'go-cache');
33+
const gomodcacheDir = path.join(runnerTemp, 'go-mod-cache');
34+
const gopathDir = path.join(runnerTemp, 'go-path');
35+
const gotmpDir = path.join(runnerTemp, 'go-tmp');
36+
37+
core.exportVariable('GOCACHE', gocacheDir);
38+
core.exportVariable('GOMODCACHE', gomodcacheDir);
39+
core.exportVariable('GOPATH', gopathDir);
40+
core.exportVariable('GOTMPDIR', gotmpDir);
41+
42+
core.setOutput('gocache', gocacheDir);
43+
core.setOutput('gomodcache', gomodcacheDir);
44+
core.setOutput('gopath', gopathDir);
45+
core.setOutput('gotmp', gotmpDir);
46+
47+
const cachedDirs = `${gocacheDir}\n${gomodcacheDir}`;
48+
core.setOutput('cached-dirs', cachedDirs);
49+
50+
- name: Create directories
51+
shell: bash
52+
run: |
53+
set -e
54+
mkdir -p "$GOCACHE"
55+
mkdir -p "$GOMODCACHE"
56+
mkdir -p "$GOPATH"
57+
mkdir -p "$GOTMPDIR"

‎.github/actions/setup-go/action.yaml

Lines changed: 8 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,42 +8,26 @@ inputs:
88
use-preinstalled-go:
99
description: "Whether to use preinstalled Go."
1010
default: "false"
11-
use-temp-cache-dirs:
12-
description: "Whether to use temporary GOCACHE and GOMODCACHE directories."
13-
default: "false"
11+
use-cache:
12+
description: "Whether to use the cache."
13+
default: "true"
1414
runs:
1515
using: "composite"
1616
steps:
17-
- name: Override GOCACHE and GOMODCACHE
18-
shell: bash
19-
if: inputs.use-temp-cache-dirs == 'true'
20-
run: |
21-
# cd to another directory to ensure we're not inside a Go project.
22-
# That'd trigger Go to download the toolchain for that project.
23-
cd "$RUNNER_TEMP"
24-
# RUNNER_TEMP should be backed by a RAM disk on Windows if
25-
# coder/setup-ramdisk-action was used
26-
export GOCACHE_DIR="$RUNNER_TEMP""\go-cache"
27-
export GOMODCACHE_DIR="$RUNNER_TEMP""\go-mod-cache"
28-
export GOPATH_DIR="$RUNNER_TEMP""\go-path"
29-
export GOTMP_DIR="$RUNNER_TEMP""\go-tmp"
30-
mkdir -p "$GOCACHE_DIR"
31-
mkdir -p "$GOMODCACHE_DIR"
32-
mkdir -p "$GOPATH_DIR"
33-
mkdir -p "$GOTMP_DIR"
34-
go env -w GOCACHE="$GOCACHE_DIR"
35-
go env -w GOMODCACHE="$GOMODCACHE_DIR"
36-
go env -w GOPATH="$GOPATH_DIR"
37-
go env -w GOTMPDIR="$GOTMP_DIR"
3817
- name: Setup Go
3918
uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
4019
with:
4120
go-version: ${{ inputs.use-preinstalled-go == 'false' && inputs.version || '' }}
21+
cache: ${{ inputs.use-cache }}
4222

4323
- name: Install gotestsum
4424
shell: bash
4525
run: go install gotest.tools/gotestsum@0d9599e513d70e5792bb9334869f82f6e8b53d4d # main as of 2025-05-15
4626

27+
- name: Install mtimehash
28+
shell: bash
29+
run: go install github.com/slsyy/mtimehash/cmd/mtimehash@a6b5da4ed2c4a40e7b805534b004e9fde7b53ce0 # v1.0.0
30+
4731
# It isn't necessary that we ever do this, but it helps
4832
# separate the "setup" from the "run" times.
4933
- name: go mod download

‎.github/workflows/ci.yaml

Lines changed: 51 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -490,21 +490,33 @@ jobs:
490490
# a separate repository to allow its use before actions/checkout.
491491
- name: Setup RAM Disks
492492
if: runner.os == 'Windows'
493-
uses: coder/setup-ramdisk-action@79dacfe70c47ad6d6c0dd7f45412368802641439
493+
uses: coder/setup-ramdisk-action@417abac44189f3f234f4fd6dd8c1ddcc8c3acd5e
494494

495495
- name: Checkout
496496
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
497497
with:
498498
fetch-depth: 1
499499

500+
- name: Setup Go Paths
501+
id: go-paths
502+
uses: ./.github/actions/setup-go-paths
503+
504+
- name: Download Go Build Cache
505+
id: download-go-build-cache
506+
uses: ./.github/actions/test-cache/download
507+
with:
508+
key-prefix: test-go-build-pr4-${{ runner.os }}-${{ runner.arch }}
509+
cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
510+
500511
- name: Setup Go
512+
id: setup-go
501513
uses: ./.github/actions/setup-go
502514
with:
503515
# Runners have Go baked-in and Go will automatically
504516
# download the toolchain configured in go.mod, so we don't
505517
# need to reinstall it. It's faster on Windows runners.
506518
use-preinstalled-go: ${{ runner.os == 'Windows' }}
507-
use-temp-cache-dirs: ${{ runner.os == 'Windows' }}
519+
use-cache: false
508520

509521
- name: Setup Terraform
510522
uses: ./.github/actions/setup-tf
@@ -515,14 +527,24 @@ jobs:
515527
with:
516528
key-prefix: test-go-pg-${{ runner.os }}-${{ runner.arch }}
517529

530+
- name: Normalize File and Directory Timestamps
531+
shell: bash
532+
run: |
533+
find . -type f ! -path ./.git/\*\* | mtimehash
534+
find . -type d ! -path ./.git/\*\* -exec touch -t 200601010000 {} +
535+
518536
- name: Test with PostgreSQL Database
519537
env:
520538
POSTGRES_VERSION: "13"
521539
TS_DEBUG_DISCO: "true"
522540
LC_CTYPE: "en_US.UTF-8"
523541
LC_ALL: "en_US.UTF-8"
542+
524543
shell: bash
525544
run: |
545+
set -o errexit
546+
set -o pipefail
547+
526548
if [ "${{ runner.os }}" == "Windows" ]; then
527549
# Create a temp dir on the R: ramdisk drive for Windows. The default
528550
# C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
@@ -533,6 +555,8 @@ jobs:
533555
mkdir -p /tmp/tmpfs
534556
sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
535557
go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg
558+
elif [ "${{ runner.os }}" == "Linux" ]; then
559+
make test-postgres-docker
536560
fi
537561
538562
# if macOS, install google-chrome for scaletests
@@ -542,10 +566,6 @@ jobs:
542566
brew install google-chrome
543567
fi
544568
545-
# By default Go will use the number of logical CPUs, which
546-
# is a fine default.
547-
PARALLEL_FLAG=""
548-
549569
# macOS will output "The default interactive shell is now zsh"
550570
# intermittently in CI...
551571
if [ "${{ matrix.os }}" == "macos-latest" ]; then
@@ -572,16 +592,33 @@ jobs:
572592
NUM_PARALLEL_TESTS=8
573593
fi
574594
575-
if [ "${{ runner.os }}" == "Linux" ]; then
576-
make test-postgres
577-
else
578-
# We rerun failing tests to counteract flakiness coming from Postgres
579-
# choking on macOS and Windows sometimes.
580-
DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
581-
--format standard-quiet --packages "./..." \
582-
-- -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS -count=1
595+
# by default, run tests with cache
596+
TESTCOUNT=""
597+
if [ "${{ github.ref }}" == "refs/heads/main" ]; then
598+
# on main, run tests without cache
599+
TESTCOUNT="-count=1"
583600
fi
584601
602+
# see scripts/normalize_path.sh for why we need this
603+
mkdir -p "$RUNNER_TEMP/sym"
604+
source scripts/normalize_path.sh
605+
export DIR_PREFIX="$(dirname $(which terraform))"
606+
normalize_path_with_symlinks "$RUNNER_TEMP/sym"
607+
608+
# We rerun failing tests to counteract flakiness coming from Postgres
609+
# choking on macOS and Windows sometimes.
610+
# TODO: for testing purposes we're only running the agent tests.
611+
# THIS MUST BE REVERTED BEFORE MERGING.
612+
DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
613+
--format standard-quiet --packages "./agent/..." \
614+
-- -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT
615+
616+
- name: Upload Go Build Cache
617+
uses: ./.github/actions/test-cache/upload
618+
with:
619+
cache-key: ${{ steps.download-go-build-cache.outputs.cache-key }}
620+
cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
621+
585622
- name: Upload Test Cache
586623
uses: ./.github/actions/test-cache/upload
587624
with:

‎scripts/normalize_path.sh

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/bash
2+
3+
# Call: normalize_path_with_symlinks [target_dir]
4+
# Normalizes the PATH environment variable by replacing each directory
5+
# with a symbolic link in target_dir. For example, if PATH is "/usr/bin:/bin",
6+
# and target_dir is /tmp, PATH will become "/tmp/0:/tmp/1", where /tmp/0 links
7+
# to /usr/bin and /tmp/1 links to /bin.
8+
#
9+
# This is useful for ensuring that PATH is consistent across CI runs and helps
10+
# with reusing the same cache across them. Many of our go tests read the PATH
11+
# variable, and if it changes between runs, the cache gets invalidated.
12+
normalize_path_with_symlinks() {
13+
local target_dir="${1:-/tmp}" # Use first argument or default to /tmp
14+
local old_path="$PATH"
15+
local -a new_parts=()
16+
local i=0
17+
18+
IFS=':' read -ra _parts <<<"$old_path"
19+
for dir in "${_parts[@]}"; do
20+
# Skip empty components that can arise from "::"
21+
[[ -z $dir ]] && continue
22+
23+
# Skip directories that don't start with $DIR_PREFIX, if $DIR_PREFIX is set
24+
if [[ -n "$DIR_PREFIX" && "$dir" != "$DIR_PREFIX"* ]]; then
25+
new_parts+=("$dir")
26+
continue
27+
fi
28+
29+
local link="$target_dir/$i"
30+
31+
# Replace any pre-existing file or link at $target_dir/$i
32+
if [[ -e $link || -L $link ]]; then
33+
rm -rf -- "$link"
34+
fi
35+
36+
# without MSYS ln will deepcopy the directory on Windows
37+
MSYS=winsymlinks:nativestrict ln -s -- "$dir" "$link"
38+
new_parts+=("$link")
39+
i=$((i + 1))
40+
done
41+
42+
export PATH
43+
PATH="$(
44+
IFS=':'
45+
echo "${new_parts[*]}"
46+
)"
47+
}

0 commit comments

Comments
 (0)