96f861a570
This script scans the whole codebase for unicode characters and errors if it finds any, unless the character is specifically whitelisted. The motivation is to protect against homoglyph attacks, invisible unicode characters, bidirectional and other control characters, and other malicious unicode usage. Given that we mostly expect to use ASCII characters in the source code, the most robust and generic fix seems to be to just ban all unicode usage. see https://trojansource.codes/ : > Compilers, interpreters, and build pipelines supporting Unicode should throw errors or warnings > for unterminated bidirectional control characters in comments or string literals, > and for identifiers with mixed-script confusable characters. > Language specifications should formally disallow unterminated bidirectional > control characters in comments and string literals. > Code editors and repository frontends should make bidirectional control characters > and mixed-script confusable characters perceptible with visual symbols or warnings. also https://github.com/maltfield/detect-malicious-unicode
357 lines
11 KiB
YAML
357 lines
11 KiB
YAML
task:
|
|
container:
|
|
image: $ELECTRUM_IMAGE
|
|
cpu: 1
|
|
memory: 2G
|
|
matrix:
|
|
- name: "unittests: Tox Python $ELECTRUM_PYTHON_VERSION"
|
|
env:
|
|
ELECTRUM_IMAGE: python:$ELECTRUM_PYTHON_VERSION
|
|
TOXENV: py3
|
|
ELECTRUM_PYTHON_NAME: python3
|
|
matrix:
|
|
- env:
|
|
ELECTRUM_PYTHON_VERSION: 3.10
|
|
- env:
|
|
ELECTRUM_PYTHON_VERSION: 3.11
|
|
- env:
|
|
ELECTRUM_PYTHON_VERSION: 3.12
|
|
- env:
|
|
ELECTRUM_PYTHON_VERSION: 3.13
|
|
- name: "unittests: Tox Python 3 debug mode"
|
|
env:
|
|
ELECTRUM_PYTHON_VERSION: 3.13
|
|
# enable additional checks:
|
|
PYTHONASYNCIODEBUG: "1"
|
|
PYTHONDEVMODE: "1"
|
|
pip_cache:
|
|
folder: ~/.cache/pip
|
|
fingerprint_script: echo $ELECTRUM_IMAGE && cat $ELECTRUM_REQUIREMENTS_CI && cat $ELECTRUM_REQUIREMENTS
|
|
version_script:
|
|
- $ELECTRUM_PYTHON_NAME --version
|
|
tag_script:
|
|
- git tag
|
|
install_script:
|
|
- apt-get update
|
|
# qml test reqs:
|
|
- apt-get -y install libgl1 libegl1 libxkbcommon0 libdbus-1-3
|
|
- pip install -r $ELECTRUM_REQUIREMENTS_CI
|
|
libsecp_build_cache:
|
|
folder: contrib/_saved_secp256k1_build
|
|
fingerprint_script: sha256sum ./contrib/make_libsecp256k1.sh
|
|
populate_script:
|
|
- apt-get -y install automake libtool
|
|
- ./contrib/make_libsecp256k1.sh
|
|
- mkdir contrib/_saved_secp256k1_build
|
|
- cp electrum/libsecp256k1.so.* contrib/_saved_secp256k1_build/
|
|
tox_script:
|
|
- export PYTHONASYNCIODEBUG
|
|
- export PYTHONDEVMODE
|
|
- export ELECTRUM_ECC_DONT_COMPILE=1
|
|
- tox
|
|
coveralls_script:
|
|
- if [ ! -z "$COVERALLS_REPO_TOKEN" ] ; then coveralls ; fi
|
|
env:
|
|
LD_LIBRARY_PATH: contrib/_saved_secp256k1_build/
|
|
ELECTRUM_REQUIREMENTS_CI: contrib/requirements/requirements-ci.txt
|
|
ELECTRUM_REQUIREMENTS: contrib/requirements/requirements.txt
|
|
# following CI_* env vars are set up for coveralls
|
|
CI_NAME: "CirrusCI"
|
|
CI_BUILD_NUMBER: $CIRRUS_BUILD_ID
|
|
CI_JOB_ID: $CIRRUS_TASK_ID
|
|
CI_BUILD_URL: "https://cirrus-ci.com/task/$CIRRUS_TASK_ID"
|
|
CI_BRANCH: $CIRRUS_BRANCH
|
|
CI_PULL_REQUEST: $CIRRUS_PR
|
|
# in addition, COVERALLS_REPO_TOKEN is set as an "override" in https://cirrus-ci.com/settings/...
|
|
depends_on:
|
|
- "linter: Flake8 Mandatory"
|
|
|
|
task:
|
|
name: "locale: upload to crowdin"
|
|
container:
|
|
image: $ELECTRUM_IMAGE
|
|
cpu: 1
|
|
memory: 1G
|
|
pip_cache:
|
|
folder: ~/.cache/pip
|
|
fingerprint_script: echo Locale && echo $ELECTRUM_IMAGE && cat $ELECTRUM_REQUIREMENTS_CI
|
|
install_script:
|
|
- apt-get update
|
|
- apt-get -y install gettext qt6-l10n-tools
|
|
- pip install -r $ELECTRUM_REQUIREMENTS_CI
|
|
- pip install requests
|
|
submodules_script:
|
|
- git submodule update --init
|
|
locale_script:
|
|
- contrib/locale/push_locale.py
|
|
env:
|
|
ELECTRUM_IMAGE: python:3.10
|
|
ELECTRUM_REQUIREMENTS_CI: contrib/requirements/requirements-ci.txt
|
|
# in addition, crowdin_api_key is set as an "override" in https://cirrus-ci.com/settings/...
|
|
depends_on:
|
|
- "unittests: Tox Python 3.10"
|
|
only_if: $CIRRUS_BRANCH == 'master'
|
|
|
|
task:
|
|
name: "Regtest functional tests"
|
|
compute_engine_instance:
|
|
image_project: cirrus-images
|
|
image: family/docker-builder
|
|
platform: linux
|
|
cpu: 1
|
|
memory: 1G
|
|
pip_cache:
|
|
folder: ~/.cache/pip
|
|
fingerprint_script: echo Regtest && echo docker_builder && cat $ELECTRUM_REQUIREMENTS
|
|
bitcoind_cache:
|
|
folder: /tmp/bitcoind
|
|
populate_script: mkdir -p /tmp/bitcoind
|
|
install_script:
|
|
- apt-get update
|
|
- apt-get -y install curl jq bc
|
|
- python3 -m pip install --user --upgrade pip
|
|
# install electrum
|
|
- export ELECTRUM_ECC_DONT_COMPILE=1 # we build manually to make caching it easier
|
|
- python3 -m pip install .[tests]
|
|
# install e-x some commits after 1.16.0 tag
|
|
- python3 -m pip install git+https://github.com/spesmilo/electrumx.git@4e66804dc0d668cd6bd4602b547e2f5b2e227e97
|
|
- "BITCOIND_VERSION=$(curl https://bitcoincore.org/en/download/ | grep -E -i --only-matching 'Latest version: [0-9\\.]+' | grep -E --only-matching '[0-9\\.]+')"
|
|
- BITCOIND_FILENAME=bitcoin-$BITCOIND_VERSION-x86_64-linux-gnu.tar.gz
|
|
- BITCOIND_PATH=/tmp/bitcoind/$BITCOIND_FILENAME
|
|
- BITCOIND_URL=https://bitcoincore.org/bin/bitcoin-core-$BITCOIND_VERSION/$BITCOIND_FILENAME
|
|
- tar -xaf $BITCOIND_PATH || (rm -f /tmp/bitcoind/* && curl --output $BITCOIND_PATH $BITCOIND_URL && tar -xaf $BITCOIND_PATH)
|
|
- cp -a bitcoin-$BITCOIND_VERSION/* /usr/
|
|
libsecp_build_cache:
|
|
folder: contrib/_saved_secp256k1_build
|
|
fingerprint_script: sha256sum ./contrib/make_libsecp256k1.sh
|
|
populate_script:
|
|
- apt-get -y install automake libtool
|
|
- ./contrib/make_libsecp256k1.sh
|
|
- mkdir contrib/_saved_secp256k1_build
|
|
- cp electrum/libsecp256k1.so.* contrib/_saved_secp256k1_build/
|
|
bitcoind_service_background_script:
|
|
- tests/regtest/run_bitcoind.sh
|
|
electrumx_service_background_script:
|
|
- tests/regtest/run_electrumx.sh
|
|
regtest_script:
|
|
- sleep 10s
|
|
- python3 -m unittest tests/regtest.py
|
|
env:
|
|
LD_LIBRARY_PATH: contrib/_saved_secp256k1_build/
|
|
ELECTRUM_REQUIREMENTS: contrib/requirements/requirements.txt
|
|
# ElectrumX exits with an error without this:
|
|
ALLOW_ROOT: 1
|
|
depends_on:
|
|
- "linter: Flake8 Mandatory"
|
|
|
|
task:
|
|
container:
|
|
image: $ELECTRUM_IMAGE
|
|
cpu: 1
|
|
memory: 1G
|
|
pip_cache:
|
|
folder: ~/.cache/pip
|
|
fingerprint_script: echo Flake8 && echo $ELECTRUM_IMAGE && cat $ELECTRUM_REQUIREMENTS
|
|
install_script:
|
|
- pip install "flake8==7.2.0" "flake8-bugbear==24.12.12"
|
|
flake8_script:
|
|
- flake8 . --count --select="$ELECTRUM_LINTERS" --ignore="$ELECTRUM_LINTERS_IGNORE" --show-source --statistics --exclude "*_pb2.py,electrum/_vendor/"
|
|
env:
|
|
ELECTRUM_IMAGE: python:3.10
|
|
ELECTRUM_REQUIREMENTS: contrib/requirements/requirements.txt
|
|
matrix:
|
|
- name: "linter: Flake8 Mandatory"
|
|
env:
|
|
# list of error codes:
|
|
# - https://flake8.pycqa.org/en/latest/user/error-codes.html
|
|
# - https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes
|
|
# - https://github.com/PyCQA/flake8-bugbear/tree/8c0e7eb04217494d48d0ab093bf5b31db0921989#list-of-warnings
|
|
ELECTRUM_LINTERS: E9,E101,E129,E273,E274,E703,E71,E722,F5,F6,F7,F8,W191,W29,B
|
|
ELECTRUM_LINTERS_IGNORE: B007,B009,B010,B019,B036,F541,F841
|
|
- name: "linter: Flake8 Non-Mandatory"
|
|
env:
|
|
ELECTRUM_LINTERS: E,F,W,C90,B
|
|
ELECTRUM_LINTERS_IGNORE: ""
|
|
allow_failures: true
|
|
|
|
task:
|
|
name: "linter: ban unicode"
|
|
container:
|
|
image: python:3.10
|
|
cpu: 1
|
|
memory: 1G
|
|
main_script:
|
|
- contrib/ban_unicode.py
|
|
|
|
# Cron jobs configured in https://cirrus-ci.com/settings/...
|
|
# - job "nightly" on branch "master" at "0 30 2 * * ?" (every day at 02:30Z)
|
|
task:
|
|
name: "build: Windows"
|
|
matrix:
|
|
- trigger_type: manual
|
|
only_if: $CIRRUS_CRON == ""
|
|
- trigger_type: automatic
|
|
only_if: $CIRRUS_CRON == "nightly"
|
|
container:
|
|
dockerfile: contrib/build-wine/Dockerfile
|
|
cpu: 1
|
|
memory: 3G
|
|
pip_cache:
|
|
folders:
|
|
- contrib/build-wine/.cache/win*/wine_pip_cache
|
|
fingerprint_script:
|
|
- echo $CIRRUS_TASK_NAME
|
|
- git ls-files -s contrib/deterministic-build/*.txt
|
|
- git ls-files -s contrib/build-wine/
|
|
build2_cache:
|
|
folders:
|
|
- contrib/build-wine/.cache/win*/build
|
|
fingerprint_script:
|
|
- echo $CIRRUS_TASK_NAME
|
|
- cat contrib/make_libsecp256k1.sh | sha256sum
|
|
- cat contrib/make_libusb.sh | sha256sum
|
|
- cat contrib/make_zbar.sh | sha256sum
|
|
- git ls-files -s contrib/build-wine/
|
|
build_script:
|
|
- cd contrib/build-wine
|
|
- ./make_win.sh
|
|
binaries_artifacts:
|
|
path: "contrib/build-wine/dist/*"
|
|
env:
|
|
CIRRUS_WORKING_DIR: /opt/wine64/drive_c/electrum
|
|
CIRRUS_DOCKER_CONTEXT: contrib/build-wine
|
|
depends_on:
|
|
- "unittests: Tox Python 3.10"
|
|
|
|
task:
|
|
name: "build: Android (QML $APK_ARCH)"
|
|
matrix:
|
|
- trigger_type: manual
|
|
only_if: $CIRRUS_CRON == ""
|
|
- trigger_type: automatic
|
|
only_if: $CIRRUS_CRON == "nightly"
|
|
timeout_in: 90m
|
|
container:
|
|
dockerfile: contrib/android/Dockerfile
|
|
cpu: 8
|
|
memory: 24G
|
|
env:
|
|
APK_ARCH: arm64-v8a
|
|
packages_tld_folder_cache:
|
|
folder: packages
|
|
fingerprint_script:
|
|
- echo $CIRRUS_TASK_NAME && cat contrib/deterministic-build/requirements.txt && cat contrib/make_packages.sh
|
|
- git ls-files -s contrib/android/
|
|
p4a_cache:
|
|
folders:
|
|
- ".buildozer/android/platform/build-$APK_ARCH/packages"
|
|
- ".buildozer/android/platform/build-$APK_ARCH/build"
|
|
fingerprint_script:
|
|
# note: should *at least* depend on Dockerfile and p4a_recipes/, but contrib/android/ is simplest
|
|
- git ls-files -s contrib/android/
|
|
- echo "qml $APK_ARCH"
|
|
build_script:
|
|
- ./contrib/android/make_apk.sh qml "$APK_ARCH" debug
|
|
binaries_artifacts:
|
|
path: "dist/*"
|
|
depends_on:
|
|
- "unittests: Tox Python 3.10"
|
|
|
|
## mac build disabled, as Cirrus CI no longer supports Intel-based mac builds
|
|
#task:
|
|
# name: "build: macOS"
|
|
# macos_instance:
|
|
# image: catalina-xcode-11.3.1
|
|
# env:
|
|
# TARGET_OS: macOS
|
|
# pip_cache:
|
|
# folder: ~/Library/Caches/pip
|
|
# fingerprint_script:
|
|
# - echo $CIRRUS_TASK_NAME
|
|
# - git ls-files -s contrib/deterministic-build/*.txt
|
|
# - git ls-files -s contrib/osx/
|
|
# build2_cache:
|
|
# folder: contrib/osx/.cache
|
|
# fingerprint_script:
|
|
# - echo $CIRRUS_TASK_NAME
|
|
# - cat contrib/make_libsecp256k1.sh | shasum -a 256
|
|
# - cat contrib/make_libusb.sh | shasum -a 256
|
|
# - cat contrib/make_zbar.sh | shasum -a 256
|
|
# - git ls-files -s contrib/osx/
|
|
# install_script:
|
|
# - git fetch --all --tags
|
|
# build_script:
|
|
# - ./contrib/osx/make_osx.sh
|
|
# sum_script:
|
|
# - ls -lah dist
|
|
# - shasum -a 256 dist/*.dmg
|
|
# binaries_artifacts:
|
|
# path: "dist/*"
|
|
|
|
task:
|
|
name: "build: AppImage"
|
|
matrix:
|
|
- trigger_type: manual
|
|
only_if: $CIRRUS_CRON == ""
|
|
- trigger_type: automatic
|
|
only_if: $CIRRUS_CRON == "nightly"
|
|
container:
|
|
dockerfile: contrib/build-linux/appimage/Dockerfile
|
|
cpu: 2
|
|
memory: 2G
|
|
pip_cache:
|
|
folder: contrib/build-linux/appimage/.cache/pip_cache
|
|
fingerprint_script:
|
|
- echo $CIRRUS_TASK_NAME
|
|
- git ls-files -s contrib/deterministic-build/*.txt
|
|
- git ls-files -s contrib/build-linux/appimage/
|
|
build2_cache:
|
|
folder: contrib/build-linux/appimage/.cache/appimage
|
|
fingerprint_script:
|
|
- echo $CIRRUS_TASK_NAME
|
|
- cat contrib/make_libsecp256k1.sh | sha256sum
|
|
- git ls-files -s contrib/build-linux/appimage/
|
|
build_script:
|
|
- ./contrib/build-linux/appimage/make_appimage.sh
|
|
binaries_artifacts:
|
|
path: "dist/*"
|
|
env:
|
|
CIRRUS_DOCKER_CONTEXT: contrib/build-linux/appimage
|
|
depends_on:
|
|
- "unittests: Tox Python 3.10"
|
|
|
|
task:
|
|
container:
|
|
dockerfile: contrib/build-linux/sdist/Dockerfile
|
|
cpu: 1
|
|
memory: 1G
|
|
pip_cache:
|
|
folder: ~/.cache/pip
|
|
fingerprint_script:
|
|
- echo $CIRRUS_TASK_NAME
|
|
- git ls-files -s contrib/deterministic-build/*.txt
|
|
- git ls-files -s contrib/build-linux/sdist/
|
|
build_script:
|
|
- ./contrib/build-linux/sdist/make_sdist.sh
|
|
binaries_artifacts:
|
|
path: "dist/*"
|
|
matrix:
|
|
- name: "build: tarball"
|
|
- name: "build: source-only tarball"
|
|
env:
|
|
OMIT_UNCLEAN_FILES: 1
|
|
depends_on:
|
|
- "unittests: Tox Python 3.10"
|
|
|
|
task:
|
|
name: "check submodules"
|
|
container:
|
|
image: python:3.10
|
|
cpu: 1
|
|
memory: 1G
|
|
fetch_script:
|
|
- git fetch --all --tags
|
|
check_script:
|
|
- ./contrib/deterministic-build/check_submodules.sh
|
|
only_if: $CIRRUS_TAG != ''
|