Skip to content

Commit ceb4b97

Browse files
hugodutkahugodutka
authored
chore: run full macos and windows pg tests in the nightly gauntlet (#18787)
This PR starts running the full test suite on Windows and macOS in the nightly gauntlet, since the regular CI only runs agent and cli tests. The full suite is too slow to be run on every PR.
1 parent f751f81 commit ceb4b97

File tree

1 file changed

+203
-0
lines changed

1 file changed

+203
-0
lines changed

‎.github/workflows/nightly-gauntlet.yaml

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
# The nightly-gauntlet runs tests that are either too flaky or too slow to block
2+
# every PR.
3+
name: nightly-gauntlet
4+
on:
5+
schedule:
6+
# Every day at 4AM
7+
- cron: "0 4 * * 1-5"
8+
workflow_dispatch:
9+
10+
permissions:
11+
contents: read
12+
13+
jobs:
14+
test-go-pg:
15+
# make sure to adjust NUM_PARALLEL_PACKAGES and NUM_PARALLEL_TESTS below
16+
# when changing runner sizes
17+
runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }}
18+
# This timeout must be greater than the timeout set by `go test` in
19+
# `make test-postgres` to ensure we receive a trace of running
20+
# goroutines. Setting this to the timeout +5m should work quite well
21+
# even if some of the preceding steps are slow.
22+
timeout-minutes: 25
23+
strategy:
24+
matrix:
25+
os:
26+
- macos-latest
27+
- windows-2022
28+
steps:
29+
- name: Harden Runner
30+
uses: step-security/harden-runner@6c439dc8bdf85cadbbce9ed30d1c7b959517bc49 # v2.12.2
31+
with:
32+
egress-policy: audit
33+
34+
# macOS indexes all new files in the background. Our Postgres tests
35+
# create and destroy thousands of databases on disk, and Spotlight
36+
# tries to index all of them, seriously slowing down the tests.
37+
- name: Disable Spotlight Indexing
38+
if: runner.os == 'macOS'
39+
run: |
40+
sudo mdutil -a -i off
41+
sudo mdutil -X /
42+
sudo launchctl bootout system /System/Library/LaunchDaemons/com.apple.metadata.mds.plist
43+
44+
# Set up RAM disks to speed up the rest of the job. This action is in
45+
# a separate repository to allow its use before actions/checkout.
46+
- name: Setup RAM Disks
47+
if: runner.os == 'Windows'
48+
uses: coder/setup-ramdisk-action@e1100847ab2d7bcd9d14bcda8f2d1b0f07b36f1b
49+
50+
- name: Checkout
51+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
52+
with:
53+
fetch-depth: 1
54+
55+
- name: Setup Go
56+
uses: ./.github/actions/setup-go
57+
with:
58+
# Runners have Go baked-in and Go will automatically
59+
# download the toolchain configured in go.mod, so we don't
60+
# need to reinstall it. It's faster on Windows runners.
61+
use-preinstalled-go: ${{ runner.os == 'Windows' }}
62+
63+
- name: Setup Terraform
64+
uses: ./.github/actions/setup-tf
65+
66+
- name: Setup Embedded Postgres Cache Paths
67+
id: embedded-pg-cache
68+
uses: ./.github/actions/setup-embedded-pg-cache-paths
69+
70+
- name: Download Embedded Postgres Cache
71+
id: download-embedded-pg-cache
72+
uses: ./.github/actions/embedded-pg-cache/download
73+
with:
74+
key-prefix: embedded-pg-${{ runner.os }}-${{ runner.arch }}
75+
cache-path: ${{ steps.embedded-pg-cache.outputs.cached-dirs }}
76+
77+
- name: Test with PostgreSQL Database
78+
env:
79+
POSTGRES_VERSION: "13"
80+
TS_DEBUG_DISCO: "true"
81+
LC_CTYPE: "en_US.UTF-8"
82+
LC_ALL: "en_US.UTF-8"
83+
shell: bash
84+
run: |
85+
set -o errexit
86+
set -o pipefail
87+
88+
if [ "${{ runner.os }}" == "Windows" ]; then
89+
# Create a temp dir on the R: ramdisk drive for Windows. The default
90+
# C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
91+
mkdir -p "R:/temp/embedded-pg"
92+
go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg" -cache "${EMBEDDED_PG_CACHE_DIR}"
93+
elif [ "${{ runner.os }}" == "macOS" ]; then
94+
# Postgres runs faster on a ramdisk on macOS too
95+
mkdir -p /tmp/tmpfs
96+
sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
97+
go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg -cache "${EMBEDDED_PG_CACHE_DIR}"
98+
elif [ "${{ runner.os }}" == "Linux" ]; then
99+
make test-postgres-docker
100+
fi
101+
102+
# if macOS, install google-chrome for scaletests
103+
# As another concern, should we really have this kind of external dependency
104+
# requirement on standard CI?
105+
if [ "${{ matrix.os }}" == "macos-latest" ]; then
106+
brew install google-chrome
107+
fi
108+
109+
# macOS will output "The default interactive shell is now zsh"
110+
# intermittently in CI...
111+
if [ "${{ matrix.os }}" == "macos-latest" ]; then
112+
touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
113+
fi
114+
115+
if [ "${{ runner.os }}" == "Windows" ]; then
116+
# Our Windows runners have 16 cores.
117+
# On Windows Postgres chokes up when we have 16x16=256 tests
118+
# running in parallel, and dbtestutil.NewDB starts to take more than
119+
# 10s to complete sometimes causing test timeouts. With 16x8=128 tests
120+
# Postgres tends not to choke.
121+
NUM_PARALLEL_PACKAGES=8
122+
NUM_PARALLEL_TESTS=16
123+
elif [ "${{ runner.os }}" == "macOS" ]; then
124+
# Our macOS runners have 8 cores. We set NUM_PARALLEL_TESTS to 16
125+
# because the tests complete faster and Postgres doesn't choke. It seems
126+
# that macOS's tmpfs is faster than the one on Windows.
127+
NUM_PARALLEL_PACKAGES=8
128+
NUM_PARALLEL_TESTS=16
129+
elif [ "${{ runner.os }}" == "Linux" ]; then
130+
# Our Linux runners have 8 cores.
131+
NUM_PARALLEL_PACKAGES=8
132+
NUM_PARALLEL_TESTS=8
133+
fi
134+
135+
# run tests without cache
136+
TESTCOUNT="-count=1"
137+
138+
DB=ci gotestsum \
139+
--format standard-quiet --packages "./..." \
140+
-- -timeout=20m -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT
141+
142+
- name: Upload Embedded Postgres Cache
143+
uses: ./.github/actions/embedded-pg-cache/upload
144+
# We only use the embedded Postgres cache on macOS and Windows runners.
145+
if: runner.OS == 'macOS' || runner.OS == 'Windows'
146+
with:
147+
cache-key: ${{ steps.download-embedded-pg-cache.outputs.cache-key }}
148+
cache-path: "${{ steps.embedded-pg-cache.outputs.embedded-pg-cache }}"
149+
150+
- name: Upload test stats to Datadog
151+
timeout-minutes: 1
152+
continue-on-error: true
153+
uses: ./.github/actions/upload-datadog
154+
if: success() || failure()
155+
with:
156+
api-key: ${{ secrets.DATADOG_API_KEY }}
157+
158+
notify-slack-on-failure:
159+
needs:
160+
- test-go-pg
161+
runs-on: ubuntu-latest
162+
if: failure() && github.ref == 'refs/heads/main'
163+
164+
steps:
165+
- name: Send Slack notification
166+
run: |
167+
curl -X POST -H 'Content-type: application/json' \
168+
--data '{
169+
"blocks": [
170+
{
171+
"type": "header",
172+
"text": {
173+
"type": "plain_text",
174+
"text": "❌ Nightly gauntlet failed",
175+
"emoji": true
176+
}
177+
},
178+
{
179+
"type": "section",
180+
"fields": [
181+
{
182+
"type": "mrkdwn",
183+
"text": "*Workflow:*\n${{ github.workflow }}"
184+
},
185+
{
186+
"type": "mrkdwn",
187+
"text": "*Committer:*\n${{ github.actor }}"
188+
},
189+
{
190+
"type": "mrkdwn",
191+
"text": "*Commit:*\n${{ github.sha }}"
192+
}
193+
]
194+
},
195+
{
196+
"type": "section",
197+
"text": {
198+
"type": "mrkdwn",
199+
"text": "*View failure:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Click here>"
200+
}
201+
}
202+
]
203+
}' ${{ secrets.CI_FAILURE_SLACK_WEBHOOK }}

0 commit comments

Comments
 (0)