Adding a smaller test surface for the perormance tests
MODIFY
.github/workflows/performance_metrics_cron.yml
.github/workflows/performance_metrics_cron.yml
@@ -60,24 +60,18 @@ jobs:
- run: echo "Currently Pulumi $(pulumi version) is installed"
- name: Install Testing Dependencies
run: make ensure
- - name: Running ${{ matrix.clouds }}${{ matrix.languages }} Tests
- run: make specific_test_set TestSet=${{ matrix.clouds }}${{ matrix.languages
- }}
+ - name: Add Test Trace DEtails
+ run: |
+ echo "PULUMI_VERSION=$(pulumi version)" >> $GITHUB_ENV
+ - name: Running Performance Matrix
+ run: make performance_test_set
strategy:
fail-fast: false
matrix:
- clouds:
- - Aws
dotnet-version:
- 3.1.301
go-version:
- 1.16.x
- languages:
- - Cs
- - Js
- - Ts
- - Py
- - Fs
node-version:
- 13.x
platform:

MODIFY
Makefile
Makefile
@@ -25,6 +25,9 @@ specific_tag_set:
echo "running $(TagSet)$(TestSet) Acceptance Tests"
cd misc/test && go test . --timeout 4h -v -count=1 -short -parallel 40 --tags=$(TagSet) --run=TestAcc$(TagSet)$(TestSet)
+performance_test_set:
+ cd misc/test && go test . --timeout 4h -v -count=1 -short -parallel 40 --tags=Performance
+
setup_test_infra:
echo "Setting up test infra"
./misc/scripts/create-ci-cluster.sh $(StackName)

MODIFY
misc/test/aws_test.go
misc/test/aws_test.go
@@ -65,21 +65,7 @@ func TestAccAwsGoFargate(t *testing.T) {
integration.ProgramTest(t, &test)
}
-func TestAccAwsGoS3Folder(t *testing.T) {
- test := getAWSBase(t).
- With(integration.ProgramTestOptions{
- Dir: path.Join(getCwd(t), "..", "..", "aws-go-s3-folder"),
- ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
- maxWait := 10 * time.Minute
- endpoint := stack.Outputs["websiteUrl"].(string)
- assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
- return assert.Contains(t, body, "Hello, world!")
- })
- },
- })
- integration.ProgramTest(t, &test)
-}
func TestAccAwsGoS3FolderComponent(t *testing.T) {
test := getAWSBase(t).
@@ -126,38 +112,6 @@ func TestAccAwsCsAssumeRole(t *testing.T) {
integration.ProgramTest(t, &test)
}
-func TestAccAwsCsS3Folder(t *testing.T) {
- test := getAWSBase(t).
- With(integration.ProgramTestOptions{
- Dir: path.Join(getCwd(t), "..", "..", "aws-cs-s3-folder"),
- ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
- maxWait := 10 * time.Minute
- endpoint := stack.Outputs["Endpoint"].(string)
- assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
- return assert.Contains(t, body, "Hello, world!")
- })
- },
- })
-
- integration.ProgramTest(t, &test)
-}
-
-func TestAccAwsFsS3Folder(t *testing.T) {
- test := getAWSBase(t).
- With(integration.ProgramTestOptions{
- Dir: path.Join(getCwd(t), "..", "..", "aws-fs-s3-folder"),
- ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
- maxWait := 10 * time.Minute
- endpoint := stack.Outputs["endpoint"].(string)
- assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
- return assert.Contains(t, body, "Hello, world!")
- })
- },
- })
-
- integration.ProgramTest(t, &test)
-}
-
func TestAccAwsJsContainers(t *testing.T) {
test := getAWSBase(t).
With(integration.ProgramTestOptions{
@@ -174,20 +128,6 @@ func TestAccAwsJsContainers(t *testing.T) {
integration.ProgramTest(t, &test)
}
-func TestAccAwsJsS3Folder(t *testing.T) {
- test := getAWSBase(t).
- With(integration.ProgramTestOptions{
- Dir: path.Join(getCwd(t), "..", "..", "aws-js-s3-folder"),
- ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
- assertHTTPResult(t, "http://"+stack.Outputs["websiteUrl"].(string), nil, func(body string) bool {
- return assert.Contains(t, body, "Hello, Pulumi!")
- })
- },
- })
-
- integration.ProgramTest(t, &test)
-}
-
func TestAccAwsJsS3FolderComponent(t *testing.T) {
test := getAWSBase(t).
With(integration.ProgramTestOptions{
@@ -306,20 +246,6 @@ func TestAccAwsGoResources(t *testing.T) {
integration.ProgramTest(t, &test)
}
-func TestAccAwsPyS3Folder(t *testing.T) {
- test := getAWSBase(t).
- With(integration.ProgramTestOptions{
- Dir: path.Join(getCwd(t), "..", "..", "aws-py-s3-folder"),
- ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
- assertHTTPResult(t, "http://"+stack.Outputs["website_url"].(string), nil, func(body string) bool {
- return assert.Contains(t, body, "Hello, Pulumi!")
- })
- },
- })
-
- integration.ProgramTest(t, &test)
-}
-
func TestAccAwsPyStepFunctions(t *testing.T) {
test := getAWSBase(t).
With(integration.ProgramTestOptions{

ADD
misc/test/performance_tests.go
+// +build Performance all
+
+package test
+
+func TestAccAwsGoS3Folder(t *testing.T) {
+ test := getAWSBase(t).
+ With(integration.ProgramTestOptions{
+ Dir: path.Join(getCwd(t), "..", "..", "aws-go-s3-folder"),
+ ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+ maxWait := 10 * time.Minute
+ endpoint := stack.Outputs["websiteUrl"].(string)
+ assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
+ return assert.Contains(t, body, "Hello, world!")
+ })
+ },
+ })
+
+ integration.ProgramTest(t, &test)
+}
+
+func TestAccAwsCsS3Folder(t *testing.T) {
+ test := getAWSBase(t).
+ With(integration.ProgramTestOptions{
+ Dir: path.Join(getCwd(t), "..", "..", "aws-cs-s3-folder"),
+ ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+ maxWait := 10 * time.Minute
+ endpoint := stack.Outputs["Endpoint"].(string)
+ assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
+ return assert.Contains(t, body, "Hello, world!")
+ })
+ },
+ })
+
+ integration.ProgramTest(t, &test)
+}
+
+func TestAccAwsFsS3Folder(t *testing.T) {
+ test := getAWSBase(t).
+ With(integration.ProgramTestOptions{
+ Dir: path.Join(getCwd(t), "..", "..", "aws-fs-s3-folder"),
+ ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+ maxWait := 10 * time.Minute
+ endpoint := stack.Outputs["endpoint"].(string)
+ assertHTTPResultWithRetry(t, endpoint, nil, maxWait, func(body string) bool {
+ return assert.Contains(t, body, "Hello, world!")
+ })
+ },
+ })
+
+ integration.ProgramTest(t, &test)
+}
+
+func TestAccAwsJsS3Folder(t *testing.T) {
+ test := getAWSBase(t).
+ With(integration.ProgramTestOptions{
+ Dir: path.Join(getCwd(t), "..", "..", "aws-js-s3-folder"),
+ ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+ assertHTTPResult(t, "http://"+stack.Outputs["websiteUrl"].(string), nil, func(body string) bool {
+ return assert.Contains(t, body, "Hello, Pulumi!")
+ })
+ },
+ })
+
+ integration.ProgramTest(t, &test)
+}
+
+func TestAccAwsPyS3Folder(t *testing.T) {
+ test := getAWSBase(t).
+ With(integration.ProgramTestOptions{
+ Dir: path.Join(getCwd(t), "..", "..", "aws-py-s3-folder"),
+ ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+ assertHTTPResult(t, "http://"+stack.Outputs["website_url"].(string), nil, func(body string) bool {
+ return assert.Contains(t, body, "Hello, Pulumi!")
+ })
+ },
+ })
+
+ integration.ProgramTest(t, &test)
+}

Adding a smaller test surface for the perormance tests