Add initial test coverage for examples
Use the Pulumi integration test framework (github.com/pulumi/pulumi/pkg/testing/integration) to run deployment lifecycle tests against a few of the examples.
ADD
.travis.yml
+if: branch = master
+language: go
+go: 1.9
+sudo: true # give us 7.5GB and >2 bursted cores.
+before_install:
+ # Dep for Go dependency management.
+ - go get -v github.com/golang/dep/cmd/dep
+ # Node.js 6.10.2 for all JavaScript code (to match AWS Lambda).
+ - nvm install v8.10
+ # Install Yarn as per https://yarnpkg.com/lang/en/docs/install-ci/#travis-tab.
+ - curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 1.2.1
+ - export PATH=$HOME/.yarn/bin:$PATH
+ # Install Pulumi
+ - curl -L https://get.pulumi.com/ | bash -s -- --version 0.14.2
+ - export PATH=$HOME/.pulumi/bin:$PATH
+install:
+ # Now restore all dependencies, after cloning, to rebuild vendor appropriately.
+ - cd misc/test
+ - dep ensure -v
+script:
+ - go test . --timeout 1h -v
+notifications:
+ webhooks: https://ufci1w66n3.execute-api.us-west-2.amazonaws.com/stage/travis

MODIFY
aws-py-webserver/README.md
aws-py-webserver/README.md
@@ -21,14 +21,6 @@ To get the correct Amazon Linux AMI for the instance size and region, a mapping
$ pulumi stack init python-webserver-testing
```
-1. Since Pulumi is in private beta, run the following to install pip packages. For more information, see [Using Pulumi PyPI Packages](https://pulumi.io/reference/python.html#pypi-packages).
-
- ```
- pip install \
- --extra-index-url https://${PULUMI_ACCESS_TOKEN}@pypi.pulumi.com/simple \
- -r requirements.txt
- ```
-
1. Set the AWS region:
```

MODIFY
aws-py-webserver/__main__.py
aws-py-webserver/__main__.py
@@ -11,9 +11,17 @@ group = ec2.SecurityGroup('web-secgrp',
ingress=[
{ 'protocol': 'tcp', 'from_port': 80, 'to_port': 80, 'cidr_blocks': ['0.0.0.0/0'] }
])
+
+user_data = """
+#!/bin/bash
+echo "Hello, World!" > index.html
+nohup python -m SimpleHTTPServer 80 &
+"""
+
server = ec2.Instance('web-server-www',
instance_type=size,
security_groups=[group.name],
+ user_data=user_data,
ami=get_linux_ami(size))
pulumi.output('public_ip', server.public_ip)

ADD
misc/test/.gitignore
+vendor
\ No newline at end of file

ADD
misc/test/Gopkg.lock
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+ branch = "master"
+ name = "github.com/Azure/go-ansiterm"
+ packages = [
+ ".",
+ "winterm"
+ ]
+ revision = "d6e3b3328b783f23731bc4d058875b0371ff8109"
+
+[[projects]]
+ name = "github.com/Nvveen/Gotty"
+ packages = ["."]
+ revision = "a8b993ba6abdb0e0c12b0125c603323a71c7790c"
+ source = "https://github.com/ijc25/Gotty"
+
+[[projects]]
+ name = "github.com/Sirupsen/logrus"
+ packages = ["."]
+ revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc"
+ version = "v1.0.5"
+
+[[projects]]
+ name = "github.com/aws/aws-sdk-go"
+ packages = [
+ "aws",
+ "aws/awserr",
+ "aws/awsutil",
+ "aws/client",
+ "aws/client/metadata",
+ "aws/corehandlers",
+ "aws/credentials",
+ "aws/credentials/ec2rolecreds",
+ "aws/credentials/endpointcreds",
+ "aws/credentials/stscreds",
+ "aws/csm",
+ "aws/defaults",
+ "aws/ec2metadata",
+ "aws/endpoints",
+ "aws/request",
+ "aws/session",
+ "aws/signer/v4",
+ "internal/sdkio",
+ "internal/sdkrand",
+ "internal/shareddefaults",
+ "private/protocol",
+ "private/protocol/eventstream",
+ "private/protocol/eventstream/eventstreamapi",
+ "private/protocol/json/jsonutil",
+ "private/protocol/jsonrpc",
+ "private/protocol/query",
+ "private/protocol/query/queryutil",
+ "private/protocol/rest",
+ "private/protocol/restxml",
+ "private/protocol/xml/xmlutil",
+ "service/cloudwatchlogs",
+ "service/s3",
+ "service/sts"
+ ]
+ revision = "16cb9f62f8a8b7af4f38725567f5ce762e1d562b"
+ version = "v1.14.21"
+
+[[projects]]
+ name = "github.com/bgentry/speakeasy"
+ packages = ["."]
+ revision = "4aabc24848ce5fd31929f7d1e4ea74d3709c14cd"
+ version = "v0.1.0"
+
+[[projects]]
+ name = "github.com/blang/semver"
+ packages = ["."]
+ revision = "2ee87856327ba09384cabd113bc6b5d174e9ec0f"
+ version = "v3.5.1"
+
+[[projects]]
+ branch = "master"
+ name = "github.com/codahale/hdrhistogram"
+ packages = ["."]
+ revision = "3a0bb77429bd3a61596f5e8a3172445844342120"
+
+[[projects]]
+ name = "github.com/davecgh/go-spew"
+ packages = ["spew"]
+ revision = "346938d642f2ec3594ed81d874461961cd0faa76"
+ version = "v1.1.0"
+
+[[projects]]
+ name = "github.com/djherbis/times"
+ packages = ["."]
+ revision = "95292e44976d1217cf3611dc7c8d9466877d3ed5"
+ version = "v1.0.1"
+
+[[projects]]
+ name = "github.com/docker/docker"
+ packages = [
+ "pkg/term",
+ "pkg/term/windows"
+ ]
+ revision = "092cba3727bb9b4a2f0e922cd6c0f93ea270e363"
+ version = "v1.13.1"
+
+[[projects]]
+ name = "github.com/emirpasic/gods"
+ packages = [
+ "containers",
+ "lists",
+ "lists/arraylist",
+ "trees",
+ "trees/binaryheap",
+ "utils"
+ ]
+ revision = "f6c17b524822278a87e3b3bd809fec33b51f5b46"
+ version = "v1.9.0"
+
+[[projects]]
+ name = "github.com/go-ini/ini"
+ packages = ["."]
+ revision = "06f5f3d67269ccec1fe5fe4134ba6e982984f7f5"
+ version = "v1.37.0"
+
+[[projects]]
+ branch = "master"
+ name = "github.com/golang/glog"
+ packages = ["."]
+ revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"
+
+[[projects]]
+ name = "github.com/golang/protobuf"
+ packages = [
+ "proto",
+ "protoc-gen-go/descriptor",
+ "ptypes",
+ "ptypes/any",
+ "ptypes/duration",
+ "ptypes/empty",
+ "ptypes/struct",
+ "ptypes/timestamp"
+ ]
+ revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
+ version = "v1.1.0"
+
+[[projects]]
+ branch = "master"
+ name = "github.com/grpc-ecosystem/grpc-opentracing"
+ packages = ["go/otgrpc"]
+ revision = "8e809c8a86450a29b90dcc9efbf062d0fe6d9746"
+
+[[projects]]
+ branch = "master"
+ name = "github.com/hashicorp/errwrap"
+ packages = ["."]
+ revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55"
+
+[[projects]]
+ branch = "master"
+ name = "github.com/hashicorp/go-multierror"
+ packages = ["."]
+ revision = "b7773ae218740a7be65057fc60b366a49b538a44"
+
+[[projects]]
+ name = "github.com/inconshreveable/mousetrap"
+ packages = ["."]
+ revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
+ version = "v1.0"
+
+[[projects]]
+ branch = "master"
+ name = "github.com/jbenet/go-context"
+ packages = ["io"]
+ revision = "d14ea06fba99483203c19d92cfcd13ebe73135f4"
+
+[[projects]]
+ name = "github.com/jmespath/go-jmespath"
+ packages = ["."]
+ revision = "0b12d6b5"
+
+[[projects]]
+ name = "github.com/kevinburke/ssh_config"
+ packages = ["."]
+ revision = "9fc7bb800b555d63157c65a904c86a2cc7b4e795"
+ version = "0.4"
+
+[[projects]]
+ branch = "master"
+ name = "github.com/mitchellh/go-homedir"
+ packages = ["."]
+ revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66"
+
+[[projects]]
+ branch = "master"
+ name = "github.com/mitchellh/go-ps"
+ packages = ["."]
+ revision = "4fdf99ab29366514c69ccccddab5dc58b8d84062"
+
+[[projects]]
+ name = "github.com/opentracing/opentracing-go"
+ packages = [
+ ".",
+ "ext",
+ "log"
+ ]
+ revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38"
+ version = "v1.0.2"
+
+[[projects]]
+ name = "github.com/pelletier/go-buffruneio"
+ packages = ["."]
+ revision = "c37440a7cf42ac63b919c752ca73a85067e05992"
+ version = "v0.2.0"
+
+[[projects]]
+ name = "github.com/pkg/errors"
+ packages = ["."]
+ revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+ version = "v0.8.0"
+
+[[projects]]
+ name = "github.com/pmezard/go-difflib"
+ packages = ["difflib"]
+ revision = "792786c7400a136282c1664665ae0a8db921c6c2"
+ version = "v1.0.0"
+
+[[projects]]
+ branch = "master"
+ name = "github.com/pulumi/pulumi"
+ packages = [
+ "pkg/apitype",
+ "pkg/backend",
+ "pkg/backend/local",
+ "pkg/diag",
+ "pkg/diag/colors",
+ "pkg/encoding",
+ "pkg/engine",
+ "pkg/operations",
+ "pkg/resource",
+ "pkg/resource/config",
+ "pkg/resource/deploy",
+ "pkg/resource/graph",
+ "pkg/resource/plugin",
+ "pkg/resource/stack",
+ "pkg/testing",
+ "pkg/testing/integration",
+ "pkg/tokens",
+ "pkg/util/cancel",
+ "pkg/util/cmdutil",
+ "pkg/util/contract",
+ "pkg/util/fsutil",
+ "pkg/util/gitutil",
+ "pkg/util/httputil",
+ "pkg/util/logging",
+ "pkg/util/mapper",
+ "pkg/util/retry",
+ "pkg/util/rpcutil",
+ "pkg/util/rpcutil/rpcerror",
+ "pkg/util/testutil",
+ "pkg/version",
+ "pkg/workspace",
+ "sdk/proto/go"
+ ]
+ revision = "015344ab0693ca433d211698eddc6a6a85588bf0"
+
+[[projects]]
+ branch = "master"
+ name = "github.com/reconquest/loreley"
+ packages = ["."]
+ revision = "2ab6b7470a54bfa9b5b0289f9b4e8fc4839838f7"
+
+[[projects]]
+ name = "github.com/sergi/go-diff"
+ packages = ["diffmatchpatch"]
+ revision = "1744e2970ca51c86172c8190fadad617561ed6e7"
+ version = "v1.0.0"
+
+[[projects]]
+ name = "github.com/spf13/cobra"
+ packages = ["."]
+ revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
+ version = "v0.0.3"
+
+[[projects]]
+ name = "github.com/spf13/pflag"
+ packages = ["."]
+ revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
+ version = "v1.0.1"
+
+[[projects]]
+ name = "github.com/src-d/gcfg"
+ packages = [
+ ".",
+ "scanner",
+ "token",
+ "types"
+ ]
+ revision = "f187355171c936ac84a82793659ebb4936bc1c23"
+ version = "v1.3.0"
+
+[[projects]]
+ name = "github.com/stretchr/testify"
+ packages = ["assert"]
+ revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
+ version = "v1.2.2"
+
+[[projects]]
+ name = "github.com/uber/jaeger-client-go"
+ packages = [
+ ".",
+ "internal/baggage",
+ "internal/spanlog",
+ "internal/throttler",
+ "log",
+ "thrift",
+ "thrift-gen/agent",
+ "thrift-gen/jaeger",
+ "thrift-gen/sampling",
+ "thrift-gen/zipkincore",
+ "transport/zipkin",
+ "utils"
+ ]
+ revision = "b043381d944715b469fd6b37addfd30145ca1758"
+ version = "v2.14.0"
+
+[[projects]]
+ name = "github.com/uber/jaeger-lib"
+ packages = ["metrics"]
+ revision = "ed3a127ec5fef7ae9ea95b01b542c47fbd999ce5"
+ version = "v1.5.0"
+
+[[projects]]
+ name = "github.com/xanzy/ssh-agent"
+ packages = ["."]
+ revision = "640f0ab560aeb89d523bb6ac322b1244d5c3796c"
+ version = "v0.2.0"
+
+[[projects]]
+ branch = "master"
+ name = "golang.org/x/crypto"
+ packages = [
+ "cast5",
+ "curve25519",
+ "ed25519",
+ "ed25519/internal/edwards25519",
+ "internal/chacha20",
+ "internal/subtle",
+ "openpgp",
+ "openpgp/armor",
+ "openpgp/elgamal",
+ "openpgp/errors",
+ "openpgp/packet",
+ "openpgp/s2k",
+ "pbkdf2",
+ "poly1305",
+ "ssh",
+ "ssh/agent",
+ "ssh/knownhosts",
+ "ssh/terminal"
+ ]
+ revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602"
+
+[[projects]]
+ branch = "master"
+ name = "golang.org/x/net"
+ packages = [
+ "context",
+ "http/httpguts",
+ "http2",
+ "http2/hpack",
+ "idna",
+ "internal/timeseries",
+ "trace"
+ ]
+ revision = "32a936f46389aa10549d60bd7833e54b01685d09"
+
+[[projects]]
+ branch = "master"
+ name = "golang.org/x/sys"
+ packages = [
+ "unix",
+ "windows"
+ ]
+ revision = "3c6ecd8f22c6f40fbeec94c000a069d7d87c7624"
+
+[[projects]]
+ name = "golang.org/x/text"
+ packages = [
+ "collate",
+ "collate/build",
+ "internal/colltab",
+ "internal/gen",
+ "internal/tag",
+ "internal/triegen",
+ "internal/ucd",
+ "language",
+ "secure/bidirule",
+ "transform",
+ "unicode/bidi",
+ "unicode/cldr",
+ "unicode/norm",
+ "unicode/rangetable"
+ ]
+ revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
+ version = "v0.3.0"
+
+[[projects]]
+ branch = "master"
+ name = "google.golang.org/genproto"
+ packages = ["googleapis/rpc/status"]
+ revision = "8b2cc369ab52e0003a878865c9372afdd6ca5c5a"
+
+[[projects]]
+ name = "google.golang.org/grpc"
+ packages = [
+ ".",
+ "balancer",
+ "balancer/base",
+ "balancer/roundrobin",
+ "codes",
+ "connectivity",
+ "credentials",
+ "encoding",
+ "encoding/proto",
+ "grpclog",
+ "internal",
+ "internal/backoff",
+ "internal/channelz",
+ "internal/grpcrand",
+ "keepalive",
+ "metadata",
+ "naming",
+ "peer",
+ "reflection",
+ "reflection/grpc_reflection_v1alpha",
+ "resolver",
+ "resolver/dns",
+ "resolver/passthrough",
+ "stats",
+ "status",
+ "tap",
+ "transport"
+ ]
+ revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8"
+ version = "v1.13.0"
+
+[[projects]]
+ name = "gopkg.in/src-d/go-billy.v4"
+ packages = [
+ ".",
+ "helper/chroot",
+ "helper/polyfill",
+ "osfs",
+ "util"
+ ]
+ revision = "83cf655d40b15b427014d7875d10850f96edba14"
+ version = "v4.2.0"
+
+[[projects]]
+ name = "gopkg.in/src-d/go-git.v4"
+ packages = [
+ ".",
+ "config",
+ "internal/revision",
+ "plumbing",
+ "plumbing/cache",
+ "plumbing/filemode",
+ "plumbing/format/config",
+ "plumbing/format/diff",
+ "plumbing/format/gitignore",
+ "plumbing/format/idxfile",
+ "plumbing/format/index",
+ "plumbing/format/objfile",
+ "plumbing/format/packfile",
+ "plumbing/format/pktline",
+ "plumbing/object",
+ "plumbing/protocol/packp",
+ "plumbing/protocol/packp/capability",
+ "plumbing/protocol/packp/sideband",
+ "plumbing/revlist",
+ "plumbing/storer",
+ "plumbing/transport",
+ "plumbing/transport/client",
+ "plumbing/transport/file",
+ "plumbing/transport/git",
+ "plumbing/transport/http",
+ "plumbing/transport/internal/common",
+ "plumbing/transport/server",
+ "plumbing/transport/ssh",
+ "storage",
+ "storage/filesystem",
+ "storage/filesystem/dotgit",
+ "storage/memory",
+ "utils/binary",
+ "utils/diff",
+ "utils/ioutil",
+ "utils/merkletrie",
+ "utils/merkletrie/filesystem",
+ "utils/merkletrie/index",
+ "utils/merkletrie/internal/frame",
+ "utils/merkletrie/noder"
+ ]
+ revision = "b23570073eaee3489e5e3d666f22ba5cbeb53243"
+ version = "v4.4.1"
+
+[[projects]]
+ name = "gopkg.in/warnings.v0"
+ packages = ["."]
+ revision = "ec4a0fea49c7b46c2aeb0b51aac55779c607e52b"
+ version = "v0.1.2"
+
+[[projects]]
+ name = "gopkg.in/yaml.v2"
+ packages = ["."]
+ revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
+ version = "v2.2.1"
+
+[solve-meta]
+ analyzer-name = "dep"
+ analyzer-version = 1
+ inputs-digest = "2811b7bbabe0f2389bed6f5bffbf43f5d298e6c14bde0a21ff18dcc49bbc7b57"
+ solver-name = "gps-cdcl"
+ solver-version = 1

ADD
misc/test/Gopkg.toml
+[[constraint]]
+ name = "github.com/pulumi/pulumi"
+ branch = "master"

ADD
misc/test/examples_test.go
+// Copyright 2016-2017, Pulumi Corporation. All rights reserved.
+
+package test
+
+import (
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "path"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/pulumi/pulumi/pkg/testing/integration"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestExamples(t *testing.T) {
+ region := os.Getenv("AWS_REGION")
+ if region == "" {
+ t.Skipf("Skipping test due to missing AWS_REGION environment variable")
+ }
+ cwd, err := os.Getwd()
+ if !assert.NoError(t, err, "expected a valid working directory: %v", err) {
+ return
+ }
+
+ base := integration.ProgramTestOptions{
+ Tracing: "https://tracing.pulumi-engineering.com/collector/api/v1/spans",
+ }
+
+ examples := []integration.ProgramTestOptions{
+ base.With(integration.ProgramTestOptions{
+ Dir: path.Join(cwd, "..", "..", "aws-js-webserver"),
+ SkipBuild: true,
+ Config: map[string]string{
+ "aws:region": "us-west-2",
+ },
+ ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+ expectHelloWorld(t, stack.Outputs["publicHostName"])
+ },
+ }),
+ base.With(integration.ProgramTestOptions{
+ Dir: path.Join(cwd, "..", "..", "aws-js-webserver-component"),
+ SkipBuild: true,
+ Config: map[string]string{
+ "aws:region": "us-west-2",
+ },
+ ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+ expectHelloWorld(t, stack.Outputs["webUrl"])
+ },
+ }),
+ // TODO[pulumi/pulumi#1606] This test is failing in CI, disabling until this issue is resolved.
+ // base.With(integration.ProgramTestOptions{
+ // Dir: path.Join(cwd, "..", "..", "aws-py-webserver"),
+ // Verbose: true,
+ // DebugLogLevel: 8,
+ // DebugUpdates: true,
+ // SkipBuild: true,
+ // Config: map[string]string{
+ // "aws:region": "us-west-2",
+ // },
+ // ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
+ // expectHelloWorld(t, stack.Outputs["public_dns"])
+ // },
+ // }),
+ }
+
+ for _, ex := range examples {
+ example := ex
+ t.Run(example.Dir, func(t *testing.T) {
+ integration.ProgramTest(t, &example)
+ })
+ }
+}
+
+func expectHelloWorld(t *testing.T, output interface{}) bool {
+ host, ok := output.(string)
+ if !assert.True(t, ok, fmt.Sprintf("expected `%s` output", output)) {
+ return false
+ }
+ // Wait to ensure startup script has time to run
+ time.Sleep(1 * time.Minute)
+ hostname := host
+ if !strings.HasPrefix(hostname, "http://") {
+ hostname = fmt.Sprintf("http://%s", host)
+ }
+ resp, err := http.Get(hostname)
+ if !assert.NoError(t, err) {
+ return false
+ }
+ defer resp.Body.Close()
+ body, err := ioutil.ReadAll(resp.Body)
+ if !assert.NoError(t, err) {
+ return false
+ }
+ return assert.Equal(t, "Hello, World!\n", string(body))
+
+}

Add initial test coverage for examples (#102)

Use the Pulumi integration test framework (github.com/pulumi/pulumi/pkg/testing/integration) to run deployment lifecycle tests against a few of the examples.