From 4a5c41166555608a4b89f3b895e3894ea52f57a4 Mon Sep 17 00:00:00 2001 From: James Houlahan Date: Tue, 11 Oct 2022 16:39:34 +0200 Subject: [PATCH] Other: Try get the CI working again (WIP) --- .gitlab-ci.yml | 272 ++----------------------------------------------- Makefile | 7 +- 2 files changed, 11 insertions(+), 268 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index de63ac0b..1cae3359 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,122 +16,36 @@ # along with ProtonMail Bridge. If not, see . --- -image: gitlab.protontech.ch:4567/go/bridge-internal:go18 +image: golang:1.18 variables: GOPRIVATE: gitlab.protontech.ch + GOMAXPROCS: $(( ${CI_TAG_CPU} / 2 )) before_script: - - eval $(ssh-agent -s) - - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null - - mkdir -p .cache/bin - - export PATH=$(pwd)/.cache/bin:$PATH - - export GOPATH="$CI_PROJECT_DIR/.cache" - - git config --global --unset-all url.git@gitlab.protontech.ch:.insteadOf - - git config --global url.https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}.insteadOf https://${CI_SERVER_HOST}/ - - make install-dev-dependencies - - git checkout . - -cache: - key: go18-gluon - paths: - - .cache - policy: pull + - git config --global url.https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}.insteadOf https://${CI_SERVER_HOST} stages: - - cache - test - - build - - check - - mirror - - -.rules-branch-and-MR-always: - rules: - - if: $CI_COMMIT_BRANCH || $CI_PIPELINE_SOURCE == "merge_request_event" - when: always - allow_failure: false - - when: never - -.rules-branch-and-MR-manual: - rules: - - if: $CI_COMMIT_BRANCH || $CI_PIPELINE_SOURCE == "merge_request_event" - when: manual - allow_failure: true - - when: never - -.rules-branch-manual-MR-always: - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: always - allow_failure: false - - if: $CI_COMMIT_BRANCH - when: manual - allow_failure: true - - when: never - -# Stage: CACHE - -# This will ensure latest dependency versions and updates the cache for -# all other following jobs which only pull the cache. -cache-push: - stage: cache - extends: - - .rules-branch-and-MR-always - script: - - echo "" - cache: - key: go18-gluon - paths: - - .cache - -# Stage: TEST lint: stage: test - extends: - - .rules-branch-and-MR-always - before_script: - - mkdir -p .cache/bin - - export PATH=$(pwd)/.cache/bin:$PATH - - export GOPATH="$CI_PROJECT_DIR/.cache" script: - - env GOMAXPROCS=$(( ${CI_TAG_CPU} / 2 )) make lint + - make lint tags: - medium -test-linux: +test: stage: test - extends: - - .rules-branch-manual-MR-always script: - - apt-get -y install pass gnupg rng-tools - # First have enough of entropy (cat /proc/sys/kernel/random/entropy_avail). - - rngd -r /dev/urandom - # Generate GPG key without password for the password manager. - - gpg --batch --yes --passphrase '' --quick-generate-key 'tester@example.com' - # Use the last created GPG ID for the password manager. - - pass init `gpg --list-keys | grep "^ " | tail -1 | tr -d '[:space:]'` - # Then finally run the tests - make test tags: - medium -test-windows: - extends: - - .build-windows-base - - .rules-branch-and-MR-always - stage: test - needs: [] - script: - - make test - test-integration: stage: test - extends: - - .rules-branch-manual-MR-always script: - - VERBOSITY=debug make -C test test + - make test-integration tags: - large @@ -140,176 +54,4 @@ dependency-updates: script: - make updates -# Stage: BUILD - -.build-base: - stage: build - needs: ["lint"] - rules: - # GODT-1833: use `=~ /qa/` after mac and windows runners are fixed - - if: $CI_JOB_NAME =~ /build-linux-qa/ && $CI_PIPELINE_SOURCE == "merge_request_event" - when: always - allow_failure: false - - if: $CI_COMMIT_BRANCH || $CI_PIPELINE_SOURCE == "merge_request_event" - when: manual - allow_failure: true - - when: never - before_script: - - mkdir -p .cache/bin - - export PATH=$(pwd)/.cache/bin:$PATH - - export GOPATH="$CI_PROJECT_DIR/.cache" - - export PATH=$PATH:$QT6DIR/bin - script: - - make build - - git diff && git diff-index --quiet HEAD - artifacts: - # Note: The latest artifacts for refs are locked against deletion, and kept - # regardless of the expiry time. Introduced in GitLab 13.0 behind a - # disabled feature flag, and made the default behavior in GitLab 13.4. - expire_in: 1 day - when: always - paths: - - bridge_*.tgz - tags: - - large - -build-linux: - extends: .build-base - image: gitlab.protontech.ch:4567/go/bridge-internal:qt6 - variables: - VCPKG_DEFAULT_BINARY_CACHE: ${CI_PROJECT_DIR}/.cache - cache: - key: linux-vcpkg - paths: - - .cache - when: 'always' - artifacts: - name: "bridge-linux-$CI_COMMIT_SHORT_SHA" - -build-linux-qa: - extends: build-linux - variables: - BUILD_TAGS: "build_qa" - artifacts: - name: "bridge-linux-qa-$CI_COMMIT_SHORT_SHA" - - -.build-darwin-base: - extends: .build-base - before_script: - - export PATH=/usr/local/bin:$PATH - - export PATH=/usr/local/opt/git/bin:$PATH - - export PATH=/usr/local/opt/make/libexec/gnubin:$PATH - - export PATH=/usr/local/opt/go@1.13/bin:$PATH - - export PATH=/usr/local/opt/gnu-sed/libexec/gnubin:$PATH - - export GOPATH=~/go - - export PATH=$GOPATH/bin:$PATH - - export CGO_CPPFLAGS='-Wno-error -Wno-nullability-completeness -Wno-expansion-to-defined -Wno-builtin-requires-header' - script: - - go version - - make build - - git diff && git diff-index --quiet HEAD - cache: {} - tags: - - macOS - -build-darwin: - extends: .build-darwin-base - artifacts: - name: "bridge-darwin-$CI_COMMIT_SHORT_SHA" - -build-darwin-qa: - extends: .build-darwin-base - variables: - BUILD_TAGS: "build_qa" - artifacts: - name: "bridge-darwin-qa-$CI_COMMIT_SHORT_SHA" - - -.build-windows-base: - extends: .build-base - before_script: - - export GOROOT=/c/Go1.18/ - - export PATH=$GOROOT/bin:$PATH - - export GOARCH=amd64 - - export GOPATH=~/go18 - - export GO111MODULE=on - - export PATH="${GOPATH}/bin:${PATH}" - - export MSYSTEM= - - export QT6DIR=/c/grrrQt/6.3.1/msvc2019_64 - - export PATH=$PATH:${QT6DIR}/bin - - export PATH="/c/Program Files/Microsoft Visual Studio/2022/Community/Common7/IDE/CommonExtensions/Microsoft/CMake/CMake/bin:$PATH" - script: - - make build - - git diff && git diff-index --quiet HEAD - tags: - - windows-bridge - -build-windows: - extends: .build-windows-base - artifacts: - name: "bridge-windows-$CI_COMMIT_SHORT_SHA" - -build-windows-qa: - extends: .build-windows-base - variables: - BUILD_TAGS: "build_qa" - artifacts: - name: "bridge-windows-qa-$CI_COMMIT_SHORT_SHA" - -# Stage: CHECK - -check-gobinsec: - stage: check - needs: ["build-linux-qa"] - extends: - - .rules-branch-manual-MR-always - cache: - key: gobinsec-cache-v3 - paths: - - ./gobinsec-cache-valid.yml - policy: pull-push - before_script: - - mkdir build - - tar -xzf bridge_linux_*.tgz -C build - - "[ ! -f ./gobinsec-cache-valid.yml ] && wget bridgeteam.protontech.ch/bridgeteam/gobinsec-cache-valid.yml" - - mv ./gobinsec-cache-valid.yml ./utils/gobinsec_update/gobinsec-cache-valid.yml - script: - - ./utils/gobinsec_update.sh - - cp ./utils/gobinsec_update/gobinsec-cache-valid.yml ./gobinsec-cache.yml - - cat ./gobinsec-cache.yml - - gobinsec -wait -cache -config utils/gobinsec_conf.yml build/bridge - - cp ./gobinsec-cache.yml ./gobinsec-cache-valid.yml # Only update cache file if gobinsec succeeds - - - -# Stage: MIRROR - -mirror-repo: - stage: mirror - only: - refs: - - master - script: - - | - cat < ~/.ssh/config - Host github.com - Hostname ssh.github.com - User git - Port 443 - ProxyCommand connect-proxy -H $http_proxy %h %p - EOF - - ssh-keyscan -t rsa ${CI_SERVER_HOST} > ~/.ssh/known_hosts - - | - cat <> ~/.ssh/known_hosts - # ssh.github.com:443 SSH-2.0-babeld-2e9d163d - [ssh.github.com]:443 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== - EOF - - echo "$mirror_key" | tr -d '\r' | ssh-add - > /dev/null - - ssh-add -l - - git clone "$CI_REPOSITORY_URL" --branch master _REPO_CLONE; - - cd _REPO_CLONE - - git remote add public $mirror_url - - git push public master - # Pushing the latest tag from master history - - git push public "$(git describe --tags --abbrev=0 || echo master)" +# TODO: PUT BACK ALL THE JOBS! JUST DID THIS FOR NOW TO GET CI WORKING AGAIN... diff --git a/Makefile b/Makefile index f7c2b035..d3965e7d 100644 --- a/Makefile +++ b/Makefile @@ -218,9 +218,10 @@ change-copyright-year: ./utils/missing_license.sh change-year test: gofiles - go test -coverprofile=/tmp/coverage.out -run=${TESTRUN} \ - ./internal/...\ - ./pkg/... + go test -v -failfast -count=1 -p=1 -coverprofile=/tmp/coverage.out -run=${TESTRUN} ./internal/... ./pkg/... + +test-integration: gofiles + go test -v -failfast -count=1 -p=1 github.com/ProtonMail/proton-bridge/v2/tests bench: go test -run '^$$' -bench=. -memprofile bench_mem.pprof -cpuprofile bench_cpu.pprof ./internal/store