From 98e62a8c39cc23406b3fe23a79fdb02ec11ab5f6 Mon Sep 17 00:00:00 2001 From: Julien Langlois Date: Fri, 12 Dec 2025 11:15:48 -0800 Subject: [PATCH 1/4] Tests --- internal/run-tests-with.yml | 267 ++++++++++++++++++------------------ internal/run-tests.yml | 151 +++++--------------- 2 files changed, 164 insertions(+), 254 deletions(-) diff --git a/internal/run-tests-with.yml b/internal/run-tests-with.yml index 22d0fb0..6bcaa46 100644 --- a/internal/run-tests-with.yml +++ b/internal/run-tests-with.yml @@ -12,9 +12,9 @@ parameters: # Name of the virtual machine image to load. image_name: "" # Name of the Qt wrapper to pip install. Should be PySide or PySide2 - qt_wrapper: PySide2==5.15.2.1 + qt_wrapper: '' # Python version to use. - python_version: 3.7 + python_version: '' # List of Python packages that need to be pip installed for testing. extra_test_dependencies: [] # Pretty name for the job. @@ -40,13 +40,10 @@ parameters: # - bash: do_something # - bash: do_something_else post_tests_steps: [] - # When set to true, the agents for all platforms are set so tests can be executed - # on all platforms. - has_unit_tests: true jobs: - job: - displayName: ${{ parameters.job_name }} Python ${{ parameters.python_version }} + displayName: ${{ parameters.job_name }} pool: vmImage: ${{ parameters.image_name }} steps: @@ -57,132 +54,132 @@ jobs: versionSpec: ${{ parameters.python_version }} displayName: Use Python ${{ parameters.python_version }} - - template: pip-install-packages.yml - parameters: - packages: - - ${{ parameters.qt_wrapper }} - - https://github.com/shotgunsoftware/tk-toolchain/archive/${{ parameters.tk_toolchain_ref }}.zip - - pytest-azurepipelines - # Inserting a parameter that is a list into another list flattens - # the result instead of nesting it. - # E.g. given a extra_test_dependencies parameter set to [Qt.py, numpy], the packages - # parameter will end up as - # - PySide2 - # - git+https://github.com/shotgunsoftware/tk-toolchain.git@${{ parameters.tk_toolchain_ref }}#egg=tk-toolchain - # - pytest-azurepipelines - # - Qt.py - # - numpy - - ${{ parameters.extra_test_dependencies }} - - - script: | - pip --version - pip list - displayName: Check installed packages - - - template: clone-repositories.yml - parameters: - repositories: - - name: tk-core - ref: ${{ parameters.tk_core_ref }} - # Nested lists are flattened, as explained above in pip-install-packages.yml. - - ${{ parameters.additional_repositories }} - - # We can't use eq(Agent.OS, 'linux') here because template expansion is done - # before the job is assigned to an agent. - - ${{ if contains(parameters.image_name, 'ubuntu') }}: - # Start xvfb on Linux so we can test with a GUI. - - bash: | - /usr/bin/Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & - echo ">>> Started Xvfb" - displayName: Start Xvfb - - - bash: sudo apt update - displayName: APT Update - - # Install libegl1 required for PySide6 - - bash: sudo apt install libegl1 - displayName: Install libegl1 - - # Repositories using our build pipeline outside of the - # shotgunsoftware organization should not be able to run our UI automation - # so do not clone the ui automation in that case. - # - # Therefore, we need to avoid including this template for repos that are - # not under the shotgunsoftware organization. - # - # FIXME: There is no way to tell if an Azure build is running under - # our organization or not. What we can do however is check the name - # of the Azure project from which this build is launched. This means a client could - # in theory have a project named Toolkit and run these commands. In practice however, - # all tests tasks require access to our secrets so they can't be used and the build will - # fail. - - # Run the tests. The task will create a simple coverage file if one is missing. - # It will include all code except for the "tests" folder. - # Note that we're hardcoding the name of the coverage file. This is important - # as "coverage combine" will combine all coverage files that match .coverage.*. - - bash: | - (test -e .coveragerc && echo ".coveragerc was found." ) || ((python -c "print('[run]\nsource=.\nomit=\n tests/*\n[report]\n\nexclude_lines =\n raise NotImplementedError')" > .coveragerc) && echo "Generated .coveragerc") - COVERAGE_FILE=.coverage.tests python -m pytest tests --cov --cov-report xml -vv - # These environment variables need to be set so Linux runs can connect - # to xvfb and to have complete logging. Each test logging output will - # be captured by pytest and displayed on failure. - env: - QT_QPA_PLATFORM: offscreen - DISPLAY: ':99.0' - TK_DEBUG: 1 - CI: 1 - # Allows to connect to a real Shotgun site during a test. Use sparingly to avoid - # slowing down automation. Using Mockgun is still the best way to have speedy tests. - TK_TOOLCHAIN_HOST: $(sg.ci.host) - TK_TOOLCHAIN_USER_LOGIN: $(sg.ci.human.login) - TK_TOOLCHAIN_USER_PASSWORD: $(sg.ci.human.password) - SHOTGUN_TEST_ENTITY_SUFFIX: '$(Agent.Name)' - displayName: Run tests - - # The post_test_steps list will be flattened into this list of steps. - - ${{ parameters.post_tests_steps }} - - # We're done, we can now upload code coverage, but first, we'll have to - # combine the results. - - - ${{ if contains(parameters.image_name, 'ubuntu') }}: - - - bash: | - python -m coverage combine - # We need to pass in -i because we had some weird coverage entries for our frameworks and apps (tk-core is not affected) - # We have an /(builtin) and /pyscript entries in the coverage for which the xml - # genearte would generate errors otherwise since these are not actual source files - python -m coverage xml -i - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - ./codecov - displayName: Upload code coverage - - - - ${{ if contains(parameters.image_name, 'macos') }}: - - - bash: | - python -m coverage combine - # We need to pass in -i because we had some weird coverage entries for our frameworks and apps (tk-core is not affected) - # We have an /(builtin) and /pyscript entries in the coverage for which the xml - # genearte would generate errors otherwise since these are not actual source files - python -m coverage xml -i - curl -Os https://uploader.codecov.io/v0.7.3/macos/codecov - chmod +x codecov - ./codecov - displayName: Upload code coverage - - - - ${{ if contains(parameters.image_name, 'windows') }}: - - - powershell: | - python -m coverage combine - # We need to pass in -i because we had some weird coverage entries for our frameworks and apps (tk-core is not affected) - # We have an /(builtin) and /pyscript entries in the coverage for which the xml - # genearte would generate errors otherwise since these are not actual source files - python -m coverage xml -i - $ProgressPreference = 'SilentlyContinue' - Invoke-WebRequest -Uri https://uploader.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe - .\codecov.exe - displayName: Upload code coverage + # - template: pip-install-packages.yml + # parameters: + # packages: + # - ${{ parameters.qt_wrapper }} + # - https://github.com/shotgunsoftware/tk-toolchain/archive/${{ parameters.tk_toolchain_ref }}.zip + # - pytest-azurepipelines + # # Inserting a parameter that is a list into another list flattens + # # the result instead of nesting it. + # # E.g. given a extra_test_dependencies parameter set to [Qt.py, numpy], the packages + # # parameter will end up as + # # - PySide2 + # # - git+https://github.com/shotgunsoftware/tk-toolchain.git@${{ parameters.tk_toolchain_ref }}#egg=tk-toolchain + # # - pytest-azurepipelines + # # - Qt.py + # # - numpy + # - ${{ parameters.extra_test_dependencies }} + + # - script: | + # pip --version + # pip list + # displayName: Check installed packages + + # - template: clone-repositories.yml + # parameters: + # repositories: + # - name: tk-core + # ref: ${{ parameters.tk_core_ref }} + # # Nested lists are flattened, as explained above in pip-install-packages.yml. + # - ${{ parameters.additional_repositories }} + + # # We can't use eq(Agent.OS, 'linux') here because template expansion is done + # # before the job is assigned to an agent. + # - ${{ if contains(parameters.image_name, 'ubuntu') }}: + # # Start xvfb on Linux so we can test with a GUI. + # - bash: | + # /usr/bin/Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & + # echo ">>> Started Xvfb" + # displayName: Start Xvfb + + # - bash: sudo apt update + # displayName: APT Update + + # # Install libegl1 required for PySide6 + # - bash: sudo apt install libegl1 + # displayName: Install libegl1 + + # # Repositories using our build pipeline outside of the + # # shotgunsoftware organization should not be able to run our UI automation + # # so do not clone the ui automation in that case. + # # + # # Therefore, we need to avoid including this template for repos that are + # # not under the shotgunsoftware organization. + # # + # # FIXME: There is no way to tell if an Azure build is running under + # # our organization or not. What we can do however is check the name + # # of the Azure project from which this build is launched. This means a client could + # # in theory have a project named Toolkit and run these commands. In practice however, + # # all tests tasks require access to our secrets so they can't be used and the build will + # # fail. + + # # Run the tests. The task will create a simple coverage file if one is missing. + # # It will include all code except for the "tests" folder. + # # Note that we're hardcoding the name of the coverage file. This is important + # # as "coverage combine" will combine all coverage files that match .coverage.*. + # - bash: | + # (test -e .coveragerc && echo ".coveragerc was found." ) || ((python -c "print('[run]\nsource=.\nomit=\n tests/*\n[report]\n\nexclude_lines =\n raise NotImplementedError')" > .coveragerc) && echo "Generated .coveragerc") + # COVERAGE_FILE=.coverage.tests python -m pytest tests --cov --cov-report xml -vv + # # These environment variables need to be set so Linux runs can connect + # # to xvfb and to have complete logging. Each test logging output will + # # be captured by pytest and displayed on failure. + # env: + # QT_QPA_PLATFORM: offscreen + # DISPLAY: ':99.0' + # TK_DEBUG: 1 + # CI: 1 + # # Allows to connect to a real Shotgun site during a test. Use sparingly to avoid + # # slowing down automation. Using Mockgun is still the best way to have speedy tests. + # TK_TOOLCHAIN_HOST: $(sg.ci.host) + # TK_TOOLCHAIN_USER_LOGIN: $(sg.ci.human.login) + # TK_TOOLCHAIN_USER_PASSWORD: $(sg.ci.human.password) + # SHOTGUN_TEST_ENTITY_SUFFIX: '$(Agent.Name)' + # displayName: Run tests + + # # The post_test_steps list will be flattened into this list of steps. + # - ${{ parameters.post_tests_steps }} + + # # We're done, we can now upload code coverage, but first, we'll have to + # # combine the results. + + # - ${{ if contains(parameters.image_name, 'ubuntu') }}: + + # - bash: | + # python -m coverage combine + # # We need to pass in -i because we had some weird coverage entries for our frameworks and apps (tk-core is not affected) + # # We have an /(builtin) and /pyscript entries in the coverage for which the xml + # # genearte would generate errors otherwise since these are not actual source files + # python -m coverage xml -i + # curl -Os https://uploader.codecov.io/latest/linux/codecov + # chmod +x codecov + # ./codecov + # displayName: Upload code coverage + + + # - ${{ if contains(parameters.image_name, 'macos') }}: + + # - bash: | + # python -m coverage combine + # # We need to pass in -i because we had some weird coverage entries for our frameworks and apps (tk-core is not affected) + # # We have an /(builtin) and /pyscript entries in the coverage for which the xml + # # genearte would generate errors otherwise since these are not actual source files + # python -m coverage xml -i + # curl -Os https://uploader.codecov.io/v0.7.3/macos/codecov + # chmod +x codecov + # ./codecov + # displayName: Upload code coverage + + + # - ${{ if contains(parameters.image_name, 'windows') }}: + + # - powershell: | + # python -m coverage combine + # # We need to pass in -i because we had some weird coverage entries for our frameworks and apps (tk-core is not affected) + # # We have an /(builtin) and /pyscript entries in the coverage for which the xml + # # genearte would generate errors otherwise since these are not actual source files + # python -m coverage xml -i + # $ProgressPreference = 'SilentlyContinue' + # Invoke-WebRequest -Uri https://uploader.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe + # .\codecov.exe + # displayName: Upload code coverage diff --git a/internal/run-tests.yml b/internal/run-tests.yml index 2df173a..d290e8f 100644 --- a/internal/run-tests.yml +++ b/internal/run-tests.yml @@ -35,6 +35,24 @@ parameters: # If set to true, the build agent will run unit tests. has_unit_tests: true + python_versions: + - '3.9' + - '3.10' + - '3.11' + + os_versions: + - name: Windows + vm_image: windows-latest + # windows-2022 + + - name: Linux + vm_image: ubuntu-latest + # ubuntu-22.04 + + - name: macOS + vm_image: macOS-latest + # macOS-14 + # TODO: At some point, we should review how these environments are enumerated. # Something like: # @@ -62,124 +80,19 @@ parameters: jobs: - ${{ if eq( parameters.has_unit_tests, true ) }}: - - template: run-tests-with.yml - parameters: - image_name: 'windows-2022' - python_version: 3.9 - job_name: "Windows" - # pass through all parameters - extra_test_dependencies: ${{ parameters.extra_test_dependencies }} - tk_toolchain_ref: ${{ parameters.tk_toolchain_ref }} - additional_repositories: ${{ parameters.additional_repositories }} - tk_core_ref: ${{ parameters.tk_core_ref }} - post_tests_steps: ${{ parameters.post_tests_steps }} - has_unit_tests: ${{ parameters.has_unit_tests }} - - - template: run-tests-with.yml - parameters: - image_name: 'macOS-14' - python_version: 3.9 - job_name: "macOS" - # pass through all parameters. - extra_test_dependencies: ${{ parameters.extra_test_dependencies }} - tk_toolchain_ref: ${{ parameters.tk_toolchain_ref }} - additional_repositories: ${{ parameters.additional_repositories }} - tk_core_ref: ${{ parameters.tk_core_ref }} - post_tests_steps: ${{ parameters.post_tests_steps }} - has_unit_tests: ${{ parameters.has_unit_tests }} - - - template: run-tests-with.yml - parameters: - image_name: 'ubuntu-22.04' - python_version: 3.9 - job_name: "Linux" - # pass through all parameters. - extra_test_dependencies: ${{ parameters.extra_test_dependencies }} - tk_toolchain_ref: ${{ parameters.tk_toolchain_ref }} - additional_repositories: ${{ parameters.additional_repositories }} - tk_core_ref: ${{ parameters.tk_core_ref }} - post_tests_steps: ${{ parameters.post_tests_steps }} - has_unit_tests: ${{ parameters.has_unit_tests }} - - # ------------------------- - - template: run-tests-with.yml - parameters: - image_name: 'windows-2022' - python_version: 3.10 - job_name: "Windows" - # pass through all parameters - extra_test_dependencies: ${{ parameters.extra_test_dependencies }} - tk_toolchain_ref: ${{ parameters.tk_toolchain_ref }} - additional_repositories: ${{ parameters.additional_repositories }} - tk_core_ref: ${{ parameters.tk_core_ref }} - post_tests_steps: ${{ parameters.post_tests_steps }} - has_unit_tests: ${{ parameters.has_unit_tests }} - - - template: run-tests-with.yml - parameters: - image_name: 'macOS-14' - python_version: 3.10 - job_name: "macOS" - # pass through all parameters. - extra_test_dependencies: ${{ parameters.extra_test_dependencies }} - tk_toolchain_ref: ${{ parameters.tk_toolchain_ref }} - additional_repositories: ${{ parameters.additional_repositories }} - tk_core_ref: ${{ parameters.tk_core_ref }} - post_tests_steps: ${{ parameters.post_tests_steps }} - has_unit_tests: ${{ parameters.has_unit_tests }} - - - template: run-tests-with.yml - parameters: - image_name: 'ubuntu-22.04' - python_version: 3.10 - job_name: "Linux" - # pass through all parameters. - extra_test_dependencies: ${{ parameters.extra_test_dependencies }} - tk_toolchain_ref: ${{ parameters.tk_toolchain_ref }} - additional_repositories: ${{ parameters.additional_repositories }} - tk_core_ref: ${{ parameters.tk_core_ref }} - post_tests_steps: ${{ parameters.post_tests_steps }} - has_unit_tests: ${{ parameters.has_unit_tests }} - - # ------------------------- - - template: run-tests-with.yml - parameters: - image_name: 'windows-2022' - python_version: 3.11 - qt_wrapper: PySide6 - job_name: "Windows" - # pass through all parameters - extra_test_dependencies: ${{ parameters.extra_test_dependencies }} - tk_toolchain_ref: ${{ parameters.tk_toolchain_ref }} - additional_repositories: ${{ parameters.additional_repositories }} - tk_core_ref: ${{ parameters.tk_core_ref }} - post_tests_steps: ${{ parameters.post_tests_steps }} - has_unit_tests: ${{ parameters.has_unit_tests }} - - template: run-tests-with.yml - parameters: - image_name: 'macOS-14' - python_version: 3.11 - qt_wrapper: PySide6 - job_name: "macOS" - # pass through all parameters. - extra_test_dependencies: ${{ parameters.extra_test_dependencies }} - tk_toolchain_ref: ${{ parameters.tk_toolchain_ref }} - additional_repositories: ${{ parameters.additional_repositories }} - tk_core_ref: ${{ parameters.tk_core_ref }} - post_tests_steps: ${{ parameters.post_tests_steps }} - has_unit_tests: ${{ parameters.has_unit_tests }} + - ${{ each os_version in parameters.os_versions }}: + - ${{ each python_version in parameters.python_versions }}: + - template: run-tests-with.yml + parameters: + job_name: ${{ parameters.job_name }} Python ${{ parameters.python_version }} + image_name: ${{ os_version.vm_image }} + python_version: ${{ python_version }} + # TODO qt_wrapper.... - - template: run-tests-with.yml - parameters: - image_name: 'ubuntu-22.04' - qt_wrapper: PySide6 - python_version: 3.11 - job_name: "Linux" - # pass through all parameters. - extra_test_dependencies: ${{ parameters.extra_test_dependencies }} - tk_toolchain_ref: ${{ parameters.tk_toolchain_ref }} - additional_repositories: ${{ parameters.additional_repositories }} - tk_core_ref: ${{ parameters.tk_core_ref }} - post_tests_steps: ${{ parameters.post_tests_steps }} - has_unit_tests: ${{ parameters.has_unit_tests }} + # pass through all parameters + extra_test_dependencies: ${{ parameters.extra_test_dependencies }} + tk_toolchain_ref: ${{ parameters.tk_toolchain_ref }} + additional_repositories: ${{ parameters.additional_repositories }} + tk_core_ref: ${{ parameters.tk_core_ref }} + post_tests_steps: ${{ parameters.post_tests_steps }} From d4dd829efcc5d9fec77e4c3d14e19c9d692389ba Mon Sep 17 00:00:00 2001 From: Julien Langlois Date: Fri, 12 Dec 2025 11:19:13 -0800 Subject: [PATCH 2/4] fixup! Tests --- internal/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/run-tests.yml b/internal/run-tests.yml index d290e8f..52ca064 100644 --- a/internal/run-tests.yml +++ b/internal/run-tests.yml @@ -85,7 +85,7 @@ jobs: - ${{ each python_version in parameters.python_versions }}: - template: run-tests-with.yml parameters: - job_name: ${{ parameters.job_name }} Python ${{ parameters.python_version }} + job_name: ${{ os_version.name }} Python ${{ python_version }} image_name: ${{ os_version.vm_image }} python_version: ${{ python_version }} # TODO qt_wrapper.... From a12d353c08d5bf7eb3262093987806de250c0c81 Mon Sep 17 00:00:00 2001 From: Julien Langlois Date: Fri, 12 Dec 2025 11:27:47 -0800 Subject: [PATCH 3/4] Tests --- internal/run-tests-with.yml | 258 ++++++++++++++++++------------------ internal/run-tests.yml | 58 +++----- 2 files changed, 148 insertions(+), 168 deletions(-) diff --git a/internal/run-tests-with.yml b/internal/run-tests-with.yml index 6bcaa46..0cdf092 100644 --- a/internal/run-tests-with.yml +++ b/internal/run-tests-with.yml @@ -54,132 +54,132 @@ jobs: versionSpec: ${{ parameters.python_version }} displayName: Use Python ${{ parameters.python_version }} - # - template: pip-install-packages.yml - # parameters: - # packages: - # - ${{ parameters.qt_wrapper }} - # - https://github.com/shotgunsoftware/tk-toolchain/archive/${{ parameters.tk_toolchain_ref }}.zip - # - pytest-azurepipelines - # # Inserting a parameter that is a list into another list flattens - # # the result instead of nesting it. - # # E.g. given a extra_test_dependencies parameter set to [Qt.py, numpy], the packages - # # parameter will end up as - # # - PySide2 - # # - git+https://github.com/shotgunsoftware/tk-toolchain.git@${{ parameters.tk_toolchain_ref }}#egg=tk-toolchain - # # - pytest-azurepipelines - # # - Qt.py - # # - numpy - # - ${{ parameters.extra_test_dependencies }} - - # - script: | - # pip --version - # pip list - # displayName: Check installed packages - - # - template: clone-repositories.yml - # parameters: - # repositories: - # - name: tk-core - # ref: ${{ parameters.tk_core_ref }} - # # Nested lists are flattened, as explained above in pip-install-packages.yml. - # - ${{ parameters.additional_repositories }} - - # # We can't use eq(Agent.OS, 'linux') here because template expansion is done - # # before the job is assigned to an agent. - # - ${{ if contains(parameters.image_name, 'ubuntu') }}: - # # Start xvfb on Linux so we can test with a GUI. - # - bash: | - # /usr/bin/Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & - # echo ">>> Started Xvfb" - # displayName: Start Xvfb - - # - bash: sudo apt update - # displayName: APT Update - - # # Install libegl1 required for PySide6 - # - bash: sudo apt install libegl1 - # displayName: Install libegl1 - - # # Repositories using our build pipeline outside of the - # # shotgunsoftware organization should not be able to run our UI automation - # # so do not clone the ui automation in that case. - # # - # # Therefore, we need to avoid including this template for repos that are - # # not under the shotgunsoftware organization. - # # - # # FIXME: There is no way to tell if an Azure build is running under - # # our organization or not. What we can do however is check the name - # # of the Azure project from which this build is launched. This means a client could - # # in theory have a project named Toolkit and run these commands. In practice however, - # # all tests tasks require access to our secrets so they can't be used and the build will - # # fail. - - # # Run the tests. The task will create a simple coverage file if one is missing. - # # It will include all code except for the "tests" folder. - # # Note that we're hardcoding the name of the coverage file. This is important - # # as "coverage combine" will combine all coverage files that match .coverage.*. - # - bash: | - # (test -e .coveragerc && echo ".coveragerc was found." ) || ((python -c "print('[run]\nsource=.\nomit=\n tests/*\n[report]\n\nexclude_lines =\n raise NotImplementedError')" > .coveragerc) && echo "Generated .coveragerc") - # COVERAGE_FILE=.coverage.tests python -m pytest tests --cov --cov-report xml -vv - # # These environment variables need to be set so Linux runs can connect - # # to xvfb and to have complete logging. Each test logging output will - # # be captured by pytest and displayed on failure. - # env: - # QT_QPA_PLATFORM: offscreen - # DISPLAY: ':99.0' - # TK_DEBUG: 1 - # CI: 1 - # # Allows to connect to a real Shotgun site during a test. Use sparingly to avoid - # # slowing down automation. Using Mockgun is still the best way to have speedy tests. - # TK_TOOLCHAIN_HOST: $(sg.ci.host) - # TK_TOOLCHAIN_USER_LOGIN: $(sg.ci.human.login) - # TK_TOOLCHAIN_USER_PASSWORD: $(sg.ci.human.password) - # SHOTGUN_TEST_ENTITY_SUFFIX: '$(Agent.Name)' - # displayName: Run tests - - # # The post_test_steps list will be flattened into this list of steps. - # - ${{ parameters.post_tests_steps }} - - # # We're done, we can now upload code coverage, but first, we'll have to - # # combine the results. - - # - ${{ if contains(parameters.image_name, 'ubuntu') }}: - - # - bash: | - # python -m coverage combine - # # We need to pass in -i because we had some weird coverage entries for our frameworks and apps (tk-core is not affected) - # # We have an /(builtin) and /pyscript entries in the coverage for which the xml - # # genearte would generate errors otherwise since these are not actual source files - # python -m coverage xml -i - # curl -Os https://uploader.codecov.io/latest/linux/codecov - # chmod +x codecov - # ./codecov - # displayName: Upload code coverage - - - # - ${{ if contains(parameters.image_name, 'macos') }}: - - # - bash: | - # python -m coverage combine - # # We need to pass in -i because we had some weird coverage entries for our frameworks and apps (tk-core is not affected) - # # We have an /(builtin) and /pyscript entries in the coverage for which the xml - # # genearte would generate errors otherwise since these are not actual source files - # python -m coverage xml -i - # curl -Os https://uploader.codecov.io/v0.7.3/macos/codecov - # chmod +x codecov - # ./codecov - # displayName: Upload code coverage - - - # - ${{ if contains(parameters.image_name, 'windows') }}: - - # - powershell: | - # python -m coverage combine - # # We need to pass in -i because we had some weird coverage entries for our frameworks and apps (tk-core is not affected) - # # We have an /(builtin) and /pyscript entries in the coverage for which the xml - # # genearte would generate errors otherwise since these are not actual source files - # python -m coverage xml -i - # $ProgressPreference = 'SilentlyContinue' - # Invoke-WebRequest -Uri https://uploader.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe - # .\codecov.exe - # displayName: Upload code coverage + - template: pip-install-packages.yml + parameters: + packages: + - ${{ parameters.qt_wrapper }} + - https://github.com/shotgunsoftware/tk-toolchain/archive/${{ parameters.tk_toolchain_ref }}.zip + - pytest-azurepipelines + # Inserting a parameter that is a list into another list flattens + # the result instead of nesting it. + # E.g. given a extra_test_dependencies parameter set to [Qt.py, numpy], the packages + # parameter will end up as + # - PySide2 + # - git+https://github.com/shotgunsoftware/tk-toolchain.git@${{ parameters.tk_toolchain_ref }}#egg=tk-toolchain + # - pytest-azurepipelines + # - Qt.py + # - numpy + - ${{ parameters.extra_test_dependencies }} + + - script: | + pip --version + pip list + displayName: Check installed packages + + - template: clone-repositories.yml + parameters: + repositories: + - name: tk-core + ref: ${{ parameters.tk_core_ref }} + # Nested lists are flattened, as explained above in pip-install-packages.yml. + - ${{ parameters.additional_repositories }} + + # We can't use eq(Agent.OS, 'linux') here because template expansion is done + # before the job is assigned to an agent. + - ${{ if contains(parameters.image_name, 'ubuntu') }}: + # Start xvfb on Linux so we can test with a GUI. + - bash: | + /usr/bin/Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & + echo ">>> Started Xvfb" + displayName: Start Xvfb + + - bash: sudo apt update + displayName: APT Update + + # Install libegl1 required for PySide6 + - bash: sudo apt install libegl1 + displayName: Install libegl1 + + # Repositories using our build pipeline outside of the + # shotgunsoftware organization should not be able to run our UI automation + # so do not clone the ui automation in that case. + # + # Therefore, we need to avoid including this template for repos that are + # not under the shotgunsoftware organization. + # + # FIXME: There is no way to tell if an Azure build is running under + # our organization or not. What we can do however is check the name + # of the Azure project from which this build is launched. This means a client could + # in theory have a project named Toolkit and run these commands. In practice however, + # all tests tasks require access to our secrets so they can't be used and the build will + # fail. + + # Run the tests. The task will create a simple coverage file if one is missing. + # It will include all code except for the "tests" folder. + # Note that we're hardcoding the name of the coverage file. This is important + # as "coverage combine" will combine all coverage files that match .coverage.*. + - bash: | + (test -e .coveragerc && echo ".coveragerc was found." ) || ((python -c "print('[run]\nsource=.\nomit=\n tests/*\n[report]\n\nexclude_lines =\n raise NotImplementedError')" > .coveragerc) && echo "Generated .coveragerc") + COVERAGE_FILE=.coverage.tests python -m pytest tests --cov --cov-report xml -vv + # These environment variables need to be set so Linux runs can connect + # to xvfb and to have complete logging. Each test logging output will + # be captured by pytest and displayed on failure. + env: + QT_QPA_PLATFORM: offscreen + DISPLAY: ':99.0' + TK_DEBUG: 1 + CI: 1 + # Allows to connect to a real Shotgun site during a test. Use sparingly to avoid + # slowing down automation. Using Mockgun is still the best way to have speedy tests. + TK_TOOLCHAIN_HOST: $(sg.ci.host) + TK_TOOLCHAIN_USER_LOGIN: $(sg.ci.human.login) + TK_TOOLCHAIN_USER_PASSWORD: $(sg.ci.human.password) + SHOTGUN_TEST_ENTITY_SUFFIX: '$(Agent.Name)' + displayName: Run tests + + # The post_test_steps list will be flattened into this list of steps. + - ${{ parameters.post_tests_steps }} + + # We're done, we can now upload code coverage, but first, we'll have to + # combine the results. + + - ${{ if contains(parameters.image_name, 'ubuntu') }}: + + - bash: | + python -m coverage combine + # We need to pass in -i because we had some weird coverage entries for our frameworks and apps (tk-core is not affected) + # We have an /(builtin) and /pyscript entries in the coverage for which the xml + # genearte would generate errors otherwise since these are not actual source files + python -m coverage xml -i + curl -Os https://uploader.codecov.io/latest/linux/codecov + chmod +x codecov + ./codecov + displayName: Upload code coverage + + + - ${{ if contains(parameters.image_name, 'macos') }}: + + - bash: | + python -m coverage combine + # We need to pass in -i because we had some weird coverage entries for our frameworks and apps (tk-core is not affected) + # We have an /(builtin) and /pyscript entries in the coverage for which the xml + # genearte would generate errors otherwise since these are not actual source files + python -m coverage xml -i + curl -Os https://uploader.codecov.io/v0.7.3/macos/codecov + chmod +x codecov + ./codecov + displayName: Upload code coverage + + + - ${{ if contains(parameters.image_name, 'windows') }}: + + - powershell: | + python -m coverage combine + # We need to pass in -i because we had some weird coverage entries for our frameworks and apps (tk-core is not affected) + # We have an /(builtin) and /pyscript entries in the coverage for which the xml + # genearte would generate errors otherwise since these are not actual source files + python -m coverage xml -i + $ProgressPreference = 'SilentlyContinue' + Invoke-WebRequest -Uri https://uploader.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe + .\codecov.exe + displayName: Upload code coverage diff --git a/internal/run-tests.yml b/internal/run-tests.yml index 52ca064..596ca62 100644 --- a/internal/run-tests.yml +++ b/internal/run-tests.yml @@ -35,60 +35,40 @@ parameters: # If set to true, the build agent will run unit tests. has_unit_tests: true - python_versions: - - '3.9' - - '3.10' - - '3.11' + vfx_reference_platform_versions: + - name: VFX CY2022 + python_version: '3.9' + qt_wrapper: PySide2==5.15.2.1 + + - name: VFX CY2023 + python_version: '3.10' + qt_wrapper: PySide2==5.15.2.1 + + - name: VFX CY2024 + python_version: '3.11' + qt_wrapper: PySide6 os_versions: - name: Windows - vm_image: windows-latest - # windows-2022 + vm_image: windows-2022 - name: Linux - vm_image: ubuntu-latest - # ubuntu-22.04 + vm_image: ubuntu-22.04 - name: macOS - vm_image: macOS-latest - # macOS-14 - -# TODO: At some point, we should review how these environments are enumerated. -# Something like: -# -# parameters: -# variants: -# Linux: -# - Qt: PySide2 -# Python: 3.7 -# - Qt: PyQt5 -# Python: 3.7 -# macOS: -# - Qt: PySide2 -# Python: 3.7 -# - Qt: PyQt5 -# Python: 3.7 -# Windows: -# - Qt: PySide2 -# Python: 3.7 -# - Qt: PyQt5 -# Python: 3.7 -# -# This would give us a better view of what variants we are running. -# Having it as a parameter would allow us also us to pick -# which builds to use for certain repositories, is necessary. + vm_image: macOS-14 + # TODO: why don't we not use the "-latest" images here like we do everywhere else? (python-api, sg-jira-bridge, ...) jobs: - ${{ if eq( parameters.has_unit_tests, true ) }}: - - ${{ each os_version in parameters.os_versions }}: - - ${{ each python_version in parameters.python_versions }}: + - ${{ each platform in parameters.vfx_reference_platform_versions }}: - template: run-tests-with.yml parameters: - job_name: ${{ os_version.name }} Python ${{ python_version }} + job_name: ${{ os_version.name }} Python ${{ platform.python_version }} image_name: ${{ os_version.vm_image }} python_version: ${{ python_version }} - # TODO qt_wrapper.... + qt_wrapper: ${{ platform.qt_wrapper }} # pass through all parameters extra_test_dependencies: ${{ parameters.extra_test_dependencies }} From 546f9d04124e3b4c705de29272be550ad094b566 Mon Sep 17 00:00:00 2001 From: Julien Langlois Date: Fri, 12 Dec 2025 11:28:45 -0800 Subject: [PATCH 4/4] fixup! Tests --- internal/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/run-tests.yml b/internal/run-tests.yml index 596ca62..215e712 100644 --- a/internal/run-tests.yml +++ b/internal/run-tests.yml @@ -67,7 +67,7 @@ jobs: parameters: job_name: ${{ os_version.name }} Python ${{ platform.python_version }} image_name: ${{ os_version.vm_image }} - python_version: ${{ python_version }} + python_version: ${{ platform.python_version }} qt_wrapper: ${{ platform.qt_wrapper }} # pass through all parameters