cloudbuild.yaml 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208
  1. #############################################################################
  2. ## The top section of this file is identical in the 3 cloudbuild.*yaml files.
  3. ## Make sure any edits you make here are copied over to the other files too
  4. ## if appropriate.
  5. ##
  6. ## TODO(al): consider if it's possible to merge these 3 files and control via
  7. ## substitutions.
  8. #############################################################################
  9. timeout: 1200s
  10. options:
  11. machineType: N1_HIGHCPU_32
  12. volumes:
  13. - name: go-modules
  14. path: /go
  15. env:
  16. - GOPROXY=https://proxy.golang.org
  17. - PROJECT_ROOT=github.com/google/certificate-transparency-go
  18. - GOPATH=/go
  19. substitutions:
  20. _CLUSTER_NAME: trillian-opensource-ci
  21. _MASTER_ZONE: us-central1-a
  22. # Cloud Build logs sent to GCS bucket
  23. logsBucket: 'gs://trillian-cloudbuild-logs'
  24. steps:
  25. # First build a "ct_testbase" docker image which contains most of the tools we need for the later steps:
  26. - name: 'gcr.io/cloud-builders/docker'
  27. entrypoint: 'bash'
  28. args: ['-c', 'docker pull gcr.io/$PROJECT_ID/ct_testbase:latest || exit 0']
  29. - name: 'gcr.io/cloud-builders/docker'
  30. args: [
  31. 'build',
  32. '-t', 'gcr.io/$PROJECT_ID/ct_testbase:latest',
  33. '--cache-from', 'gcr.io/$PROJECT_ID/ct_testbase:latest',
  34. '-f', './integration/Dockerfile',
  35. '.'
  36. ]
  37. # prepare spins up an ephemeral trillian instance for testing use.
  38. - name: gcr.io/$PROJECT_ID/ct_testbase
  39. entrypoint: 'bash'
  40. id: 'prepare'
  41. args:
  42. - '-exc'
  43. - |
  44. # Use latest versions of Trillian docker images built by the Trillian CI cloudbuilders.
  45. docker pull gcr.io/$PROJECT_ID/log_server:latest
  46. docker tag gcr.io/$PROJECT_ID/log_server:latest deployment_trillian-log-server
  47. docker pull gcr.io/$PROJECT_ID/log_signer:latest
  48. docker tag gcr.io/$PROJECT_ID/log_signer:latest deployment_trillian-log-signer
  49. # Bring up an ephemeral trillian instance using the docker-compose config in the Trillian repo:
  50. export TRILLIAN_LOCATION="$$(go list -f '{{.Dir}}' github.com/google/trillian)"
  51. # We need to fix up Trillian's docker-compose to connect to the CloudBuild network to that tests can use it:
  52. echo -e "networks:\n default:\n external:\n name: cloudbuild" >> $${TRILLIAN_LOCATION}/examples/deployment/docker-compose.yml
  53. docker-compose -f $${TRILLIAN_LOCATION}/examples/deployment/docker-compose.yml pull mysql trillian-log-server trillian-log-signer
  54. docker-compose -f $${TRILLIAN_LOCATION}/examples/deployment/docker-compose.yml up -d mysql trillian-log-server trillian-log-signer
  55. # Install proto related bits and block on Trillian being ready
  56. - name: gcr.io/$PROJECT_ID/ct_testbase
  57. id: 'ci-ready'
  58. entrypoint: 'bash'
  59. args:
  60. - '-ec'
  61. - |
  62. go install \
  63. github.com/golang/protobuf/proto \
  64. github.com/golang/protobuf/protoc-gen-go \
  65. github.com/golang/mock/mockgen \
  66. go.etcd.io/etcd/v3 go.etcd.io/etcd/etcdctl/v3 \
  67. github.com/fullstorydev/grpcurl/cmd/grpcurl
  68. # Generate all protoc and mockgen files
  69. go generate -run="protoc" ./...
  70. go generate -run="mockgen" ./...
  71. # Cache all the modules we'll need too
  72. go mod download
  73. go test ./...
  74. # Wait for trillian logserver to be up
  75. until nc -z deployment_trillian-log-server_1 8090; do echo .; sleep 5; done
  76. waitFor: ['prepare']
  77. # Run the presubmit tests
  78. - name: gcr.io/$PROJECT_ID/ct_testbase
  79. id: 'default_test'
  80. env:
  81. - 'GOFLAGS='
  82. - 'PRESUBMIT_OPTS=--no-linters --no-generate'
  83. - 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
  84. - 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
  85. waitFor: ['ci-ready']
  86. - name: gcr.io/$PROJECT_ID/ct_testbase
  87. id: 'race_detection'
  88. env:
  89. - 'GOFLAGS=-race'
  90. - 'PRESUBMIT_OPTS=--no-linters --no-generate'
  91. - 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
  92. - 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
  93. waitFor: ['ci-ready']
  94. - name: gcr.io/$PROJECT_ID/ct_testbase
  95. id: 'etcd_with_coverage'
  96. env:
  97. - 'GOFLAGS='
  98. - 'PRESUBMIT_OPTS=--no-linters --no-generate --coverage'
  99. - 'WITH_ETCD=true'
  100. - 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
  101. - 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
  102. waitFor: ['ci-ready']
  103. - name: gcr.io/$PROJECT_ID/ct_testbase
  104. id: 'etcd_with_race'
  105. env:
  106. - 'GOFLAGS=-race'
  107. - 'PRESUBMIT_OPTS=--no-linters --no-generate'
  108. - 'WITH_ETCD=true'
  109. - 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
  110. - 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
  111. waitFor: ['ci-ready']
  112. - name: gcr.io/$PROJECT_ID/ct_testbase
  113. id: 'with_pkcs11_and_race'
  114. env:
  115. - 'GOFLAGS=-race --tags=pkcs11'
  116. - 'PRESUBMIT_OPTS=--no-linters --no-generate'
  117. - 'WITH_PKCS11=true'
  118. - 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
  119. - 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
  120. waitFor: ['ci-ready']
  121. # Collect and submit codecoverage reports
  122. - name: 'gcr.io/cloud-builders/curl'
  123. id: 'codecov.io'
  124. entrypoint: bash
  125. args: ['-c', 'bash <(curl -s https://codecov.io/bash)']
  126. env:
  127. - 'VCS_COMMIT_ID=$COMMIT_SHA'
  128. - 'VCS_BRANCH_NAME=$BRANCH_NAME'
  129. - 'VCS_PULL_REQUEST=$_PR_NUMBER'
  130. - 'CI_BUILD_ID=$BUILD_ID'
  131. - 'CODECOV_TOKEN=$_CODECOV_TOKEN' # _CODECOV_TOKEN is specified in the cloud build trigger
  132. waitFor: ['etcd_with_coverage']
  133. - name: gcr.io/$PROJECT_ID/ct_testbase
  134. id: 'ci_complete'
  135. entrypoint: /bin/true
  136. waitFor: ['codecov.io', 'default_test', 'race_detection', 'etcd_with_coverage', 'etcd_with_race', 'with_pkcs11_and_race']
  137. ############################################################################
  138. ## End of replicated section.
  139. ## Below are deployment specific steps for the CD env.
  140. ############################################################################
  141. - id: build_ctfe
  142. name: gcr.io/cloud-builders/docker
  143. args:
  144. - build
  145. - --file=trillian/examples/deployment/docker/ctfe/Dockerfile
  146. - --tag=gcr.io/${PROJECT_ID}/ctfe:${COMMIT_SHA}
  147. - --cache-from=gcr.io/${PROJECT_ID}/ctfe
  148. - .
  149. waitFor: [-]
  150. - id: build_envsubst
  151. name: gcr.io/cloud-builders/docker
  152. args:
  153. - build
  154. - trillian/examples/deployment/docker/envsubst
  155. - -t
  156. - envsubst
  157. waitFor: ['ci_complete']
  158. - id: envsubst_kubernetes_configs
  159. name: envsubst
  160. args:
  161. - trillian/examples/deployment/kubernetes/ctfe-deployment.yaml
  162. - trillian/examples/deployment/kubernetes/ctfe-service.yaml
  163. - trillian/examples/deployment/kubernetes/ctfe-ingress.yaml
  164. env:
  165. - PROJECT_ID=${PROJECT_ID}
  166. - IMAGE_TAG=${COMMIT_SHA}
  167. waitFor:
  168. - build_envsubst
  169. - id: update_kubernetes_configs_dryrun
  170. name: gcr.io/cloud-builders/kubectl
  171. args:
  172. - apply
  173. - --dry-run=server
  174. - -f=trillian/examples/deployment/kubernetes/ctfe-deployment.yaml
  175. - -f=trillian/examples/deployment/kubernetes/ctfe-service.yaml
  176. - -f=trillian/examples/deployment/kubernetes/ctfe-ingress.yaml
  177. env:
  178. - CLOUDSDK_COMPUTE_ZONE=${_MASTER_ZONE}
  179. - CLOUDSDK_CONTAINER_CLUSTER=${_CLUSTER_NAME}
  180. waitFor:
  181. - envsubst_kubernetes_configs
  182. - build_ctfe
  183. images:
  184. - gcr.io/${PROJECT_ID}/ctfe:${COMMIT_SHA}
  185. - gcr.io/${PROJECT_ID}/ct_testbase:latest