From c238ad0cad9b92f02b94e4202d3431c61424c4e3 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Fri, 28 Feb 2025 10:51:48 +0100 Subject: [PATCH 01/39] Removed downloading of utr and using preinstalled version --- .yamato/console-standalone-test.yml | 6 ++--- .yamato/desktop-standalone-tests.yml | 40 +++------------------------- .yamato/mobile-standalone-test.yml | 24 ++++------------- .yamato/package-tests.yml | 9 +------ .yamato/performance-tests.yml | 20 +------------- .yamato/webgl-build.yml | 15 +---------- 6 files changed, 13 insertions(+), 101 deletions(-) diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index 2d5ca4861e..4ab96d8273 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -25,10 +25,9 @@ build_{{ project.name }}_{{ platform.name }}_{{ editor }}: # Installing tools. unity-downloader-cli and utr should be already preinstalled on the image - sudo pip install unity-downloader-cli - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --fast --wait - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat # Platform specific Build - - utr --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 + - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 variables: # PS4 related @@ -70,10 +69,9 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: # Installing tools. - sudo pip install unity-downloader-cli - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --fast --wait - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat # Platform specific Execution - - utr --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 + - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 variables: # PS4 related diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index b12c71f2eb..4d8fc8167f 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -20,32 +20,11 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ image: {{ platform.image }} flavor: {{ platform.flavor }} commands: -# Installing tools -{% if platform.name == "ubuntu" %} - - sudo apt-get update -q - - sudo apt install -qy imagemagick -{% endif %} - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - -# Platform specific UTR setup - - | -{% if platform.name == "win" %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat -{% else %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr && chmod +x utr -{% endif %} - -# Installing editor - - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} --fast --wait + - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} --fast --wait # Installing editor # Build Player - - | -{% if platform.name == "win" %} - utr.bat --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 -{% else %} - ./utr --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 -{% endif %} - + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: paths: @@ -76,24 +55,11 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ image: {{ platform.image }} flavor: {{ platform.flavor }} commands: -# Platform specific UTR setup - - | -{% if platform.name == "win" %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat -{% else %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr && chmod +x utr -{% endif %} - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} --fast --wait # Run Standalone tests - - | -{% if platform.name == "win" %} - utr.bat --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 -{% else %} - ./utr --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 -{% endif %} + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 artifacts: logs: diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index 2ca2444b86..e8ac07823a 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -21,21 +21,10 @@ build_{{ project.name }}_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - # Installing tools - - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - - # Installing editor - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} --fast --wait + - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple # Installing unity-downloader-cli + - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} --fast --wait # Installing editor - # Platform specific Build -{% if platform.base == "win" %} - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat - - utr.bat --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 -{% else %} - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr - - chmod +x ./utr - - ./utr --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 -{% endif %} + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: paths: @@ -76,7 +65,6 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: {% if platform.standalone == "Android" %} # Download standalone UnityTestRunner and ADB setup - - command: curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat - command: wget http://artifactory-slo.bf.unity3d.com/artifactory/mobile-generic/android/ADBKeys.zip!/adbkey.pub -O %USERPROFILE%/.android/adbkey.pub - command: wget http://artifactory-slo.bf.unity3d.com/artifactory/mobile-generic/android/ADBKeys.zip!/adbkey -O %USERPROFILE%/.android/adbkey - command: gsudo NetSh Advfirewall set allprofiles state off @@ -87,13 +75,11 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: # Run tests - | set ANDROID_DEVICE_CONNECTION=%BOKKEN_DEVICE_IP% - utr --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 + UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 {% else %} - # Download standalone UnityTestRunner - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr && chmod +x utr # Run tests - - ./utr --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 {% endif %} artifacts: logs: diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index 79815ebc16..c4172797f8 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -17,20 +17,13 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: UNITY_EXT_LOGGING: 1 commands: - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait - # Platform specific UTR setup - - | -{% if platform.name == "win" %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat -{% else %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr && chmod +x utr -{% endif %} # Validate packages. - upm-pvp test --unity .Editor --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --results pvp-results - upm-pvp require {% if platform.name == "win" %}"%XRAY_PROFILE%"{% else %}"$XRAY_PROFILE"{% endif %} --results pvp-results # Run UTR to test packages. - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --unity .Editor - - {% if platform.name == "ubuntu" %}DISPLAY=:0 {% endif %} {% if platform.name == "win" %} utr.bat {% else %} ./utr {% endif %} --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --extra-editor-arg=-testCategory --extra-editor-arg=!Performance --reruncount=1 --clean-library-on-rerun + - {% if platform.name == "ubuntu" %}DISPLAY=:0 {% endif %} UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --extra-editor-arg=-testCategory --extra-editor-arg=!Performance --reruncount=1 --clean-library-on-rerun artifacts: logs: paths: diff --git a/.yamato/performance-tests.yml b/.yamato/performance-tests.yml index abdbef444e..cecf3aed95 100644 --- a/.yamato/performance-tests.yml +++ b/.yamato/performance-tests.yml @@ -13,32 +13,14 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - # Installing tools -{% if platform.name == "ubuntu" %} - - sudo apt-get update -q - - sudo apt install -qy imagemagick -{% endif %} - npm install -g upm-ci-utils@stable --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - # Platform specific UTR setup -{% if platform.name == "win" %} - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat -{% else %} - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr && chmod +x utr -{% endif %} - # Installing editor - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait # Build Player - - | -{% if platform.name == "win" %} - utr.bat --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata -{% else %} - ./utr --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata -{% endif %} - + - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata artifacts: logs: paths: diff --git a/.yamato/webgl-build.yml b/.yamato/webgl-build.yml index 15db4b16ec..ef985e0cb3 100644 --- a/.yamato/webgl-build.yml +++ b/.yamato/webgl-build.yml @@ -20,25 +20,12 @@ webgl_build_{{ project.name }}_{{ platform }}_{{ editor }}: commands: # Installing tools - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - - # Platform specific UTR setup - - | -{% if platform.name == "win" %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat -{% else %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr && chmod +x utr -{% endif %} # Installing editor - unity-downloader-cli -u {{ editor }} -c Editor -c webgl -c il2cpp -w --fast # Build Player - - | -{% if platform.name == "win" %} - utr.bat --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg="-cloudEnvironment staging" --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 -{% else %} - ./utr --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg="-cloudEnvironment staging" --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 -{% endif %} + - UnifiedTestRunner --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg="-cloudEnvironment staging" --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: logs: From 92bc678e3f132aa3412972be87055e4cc9966f90 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Fri, 28 Feb 2025 11:04:38 +0100 Subject: [PATCH 02/39] Code Coverage test with upm-pvp --- .yamato/code-coverage.yml | 2 +- .yamato/package-pack.yml | 24 ------------------------ 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index c0aa7e59fd..2ef01fb7f9 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -23,6 +23,6 @@ code_coverage_{{ platform.name }}_{{ editor }}: - "upm-ci~/test-results/**/*" - "upm-ci~/test-results/CoverageResults/**/*" dependencies: - - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }}_upmCI + - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} \ No newline at end of file diff --git a/.yamato/package-pack.yml b/.yamato/package-pack.yml index 492c08ef78..a9ab902847 100644 --- a/.yamato/package-pack.yml +++ b/.yamato/package-pack.yml @@ -25,28 +25,4 @@ package_pack_-_ngo_{{ platform.name }}: packages: paths: - "upm-ci~/**" -{% endfor -%} - - -# This is in essence the same job as the one above with the difference that upm-ci is used instead of upm-pvp -# The reason for using it is that I had some problems with Code Coverage which in its current form uses upm-ci but if we would use the other pack job (the one above) we would need to use upm-pvp -# I had some problems with getting it to work so as temporary solution I created this pack job which is used ONLY as a dependency of Code Coverage job (other jobs use the above definition of pack job) -# TODO: remove this job and utilize the above one for Code Coverage job. This is tracked in MTT-11383 -{% for platform in test_platforms.default -%} -{% for project in projects.default -%} -package_pack_-_ngo_{{ platform.name }}_upmCI: - name: Package Pack (legacy upm-ci) - NGO [{{ platform.name }}] - agent: - type: {{ platform.type }} - image: {{ platform.image }} - flavor: {{ platform.flavor }} - timeout: 0.25 - commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - - upm-ci project pack --project-path {{ project.path }} - artifacts: - packages: - paths: - - "upm-ci~/packages/**/*" -{% endfor -%} {% endfor -%} \ No newline at end of file From 637c89b098465bc05a1afd9073ef3eb9d4f847fa Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Mon, 3 Mar 2025 10:23:54 +0100 Subject: [PATCH 03/39] Modified Code Coverage job to use upm-pvp instead of upm-ci --- .yamato/code-coverage.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 2ef01fb7f9..8cc6a4a14f 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -14,14 +14,16 @@ code_coverage_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - npm install -g upm-ci-utils@stable --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait - - DISPLAY=:0 upm-ci package test -u {{ editor }} --package-path com.unity.netcode.gameobjects --enable-code-coverage --code-coverage-options 'generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime' --extra-utr-arg="--extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun" + - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage --code-coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results artifacts: logs: paths: - - "upm-ci~/test-results/**/*" - - "upm-ci~/test-results/CoverageResults/**/*" + - "test-results/**/*" + - "test-results/CodeCoverage/**/*" + - "test-results/CoverageResults/**/*" + - "**/**/*" dependencies: - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} From 677a3189e491f761e9800e6c81d782fe64930aaf Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Mon, 3 Mar 2025 12:42:49 +0100 Subject: [PATCH 04/39] Added coverage paths --- .yamato/code-coverage.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 8cc6a4a14f..e1064b8629 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -1,4 +1,4 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- # Runs package tests in order to determine code coverage of the NGO package. @@ -16,14 +16,14 @@ code_coverage_{{ platform.name }}_{{ editor }}: commands: - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage --code-coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage --coverage-results-path=test-results/CodeCoverage --code-coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results artifacts: logs: paths: - - "test-results/**/*" - "test-results/CodeCoverage/**/*" - "test-results/CoverageResults/**/*" - - "**/**/*" + - "test-results/**/coverage.xml" + - "test-results/**/Report/**/*" dependencies: - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} From 030fc675d4251aed5d155641bb57e8151e7641e6 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Mon, 3 Mar 2025 14:15:06 +0100 Subject: [PATCH 05/39] tried different path for coverage results --- .yamato/code-coverage.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index e1064b8629..5838bb8a6e 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -16,14 +16,13 @@ code_coverage_{{ platform.name }}_{{ editor }}: commands: - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage --coverage-results-path=test-results/CodeCoverage --code-coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage --coverage-results-path=upm-ci~/test-results/CodeCoverage --code-coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results artifacts: logs: paths: - - "test-results/CodeCoverage/**/*" + - "upm-ci~/test-results/CodeCoverage/**/*" - "test-results/CoverageResults/**/*" - "test-results/**/coverage.xml" - - "test-results/**/Report/**/*" dependencies: - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} From 8bfcb71ce4bd5e1046aa9313947712692aa93e64 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Mon, 3 Mar 2025 14:16:28 +0100 Subject: [PATCH 06/39] Added comments and modifications to webgl job --- .yamato/webgl-build.yml | 75 +++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/.yamato/webgl-build.yml b/.yamato/webgl-build.yml index ef985e0cb3..507dffd349 100644 --- a/.yamato/webgl-build.yml +++ b/.yamato/webgl-build.yml @@ -1,40 +1,57 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Builds a player on WebGl standalone platform without executing any tests. -# This setup performs build-only validation since WebGL runs in browser and for tests to be executed we would need to -# consider having a web server, browser automation and overall complex test setup. -# Default project (testptoject) in this case is used as a context. -# WebGL requires Il2cpp scripting backend so mono is not considered -# ARM64 architectures are not considered since Windows_arm64 is recommended to use only after builds and when it comes to macOS_arm64 there is problem with OpenCL not being available -# Builds are made on each desktop platform as in project.metafile declaration -{% for project in projects.default -%} -{% for platform in test_platforms.desktop -%} -{% for editor in validation_editors.all -%} -webgl_build_{{ project.name }}_{{ platform }}_{{ editor }}: - name: WebGl Build - {{ project.name }} [{{ platform.name }}, {{ editor }}, il2cpp] - agent: - type: {% if platform.name == "mac" %} {{ platform.type }} {% else %} {{ platform.type }}::GPU {% endif %} - image: {{ platform.image }} - flavor: {{ platform.flavor }} - commands: - # Installing tools - - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple +# DESCRIPTION-------------------------------------------------------------------------- + # This job definition creates job configurations to test building a player on WebGl standalone platform. + # This job will not execute any tests, we only test building process. + # This setup performs build-only validation (without running tests) because WebGL runs in browser and for tests to be executed we would need to consider having a web server, browser automation and overall complex test setup which currently we don't have. - # Installing editor - - unity-downloader-cli -u {{ editor }} -c Editor -c webgl -c il2cpp -w --fast + # The setup below defines multiple jobs with different configurations (taken from project.metafile in the 1st line). + # This is a Yamato specific as it will "compile" this script and generate given configurations which we can easily modify from within project.metafile. + # From there (project.metafile) we could easily add for example a new validation editor or remove unsupported one. - # Build Player - - UnifiedTestRunner --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg="-cloudEnvironment staging" --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated using nested loops through: + # 1. Projects (from project.metafile) + # 2. Desktop platforms (Windows, Ubuntu, macOS) + # 3. Supported Unity Editor versions + # This structure allows easy modification of supported configurations via project.metafile + +# TECHNICAL CONSTRAINTS---------------------------------------------------------------- + # WebGL requires IL2CPP scripting backend (Mono not supported) + # x64 architectures preferred for build phase (ARM64 not used to optimize resource usage. For build phase x64 architectures are preferred) + # Build validation only (no runtime testing) + +#-------------------------------------------------------------------------------------- + +{% for project in projects.default -%} # Default project (testptoject) in this case is used as a context for the build. +{% for platform in test_platforms.desktop -%} # Jobs are being defined per each desktop platform as in project.metafile declaration (Win, Ubuntu, Mac) +{% for editor in validation_editors.all -%} # Jobs are being defined per each supported editor (For NGOv2.X it means 6000+) +webgl_build_{{ project.name }}_{{ platform }}_{{ editor }}: # This is a job name when querying from code (for example in _run-all.yml). Notice that parts of it will be filled in from for loops. For example there will be a configuration "webgl_build_testproject_win_6000.0" and when querying for this job we could for example change some values to get a given job (for example change editor value) and this will query different job configuration + name: WebGl Build - {{ project.name }} [{{ platform.name }}, {{ editor }}, il2cpp] # This is a job name visible in Yamato (notice that here we also use "generated" fields from for loops + agent: + type: {{ platform.type }} # Defines type from platform configuration (for example Unity::VM). For webgl builds no GPU extension is needed + image: {{ platform.image }} # Bokken image specification + flavor: {{ platform.flavor }} # Memory/CPU allocation configuration + commands: + - unity-downloader-cli -u {{ editor }} -c Editor -c webgl -c il2cpp # unity-downloader-cli should be already preinstalled on the image. This steps downloads proper editor to build from together with required components (webgl + il2cpp). + + # The following step builds the player with defined options such as: + # Suite parameter if defined since it's a mandatory field to define which test suite should be used, but it doesn't matter in this case since we won't run any tests (--suite) + # We specify build platform (--platform), scripting backend (--scripting-backend) and preinstalled editor location (--editor-location) + # We will be building project defined in (--testproject) + # Custom artifact paths for logs and build output (via --artifacts_path and --player-save-path) + # Editor is run in batchmode, which means that Unity runs command line arguments without the need for human interaction. It also suppresses pop-up windows that require human interaction (such as the Save Scene window). We should always run Unity in batch mode when using command line arguments, because it allows automation to run without interruption. (--extra-editor-arg=-batchmode) + # Engine is initialized in ‘nographics’ mode since we don't need any graphics for this case (--extra-editor-arg=-nographics) + # In case of failure the job will be rerunned once (--reruncount=1) with clean library (--clean-library-on-rerun) + # This will perform only building phase (--build-only) with a default timeout of 3m (--timeout=1800) + - UnifiedTestRunner --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: logs: paths: - - "artifacts/**/*" - - "build/players/**/*" - variables: - CI: true - ENABLE_BURST_COMPILATION: False + - "artifacts/**/*" # Build process logs (path defined via --artifacts_path argument) + - "build/players/**/*" # WebGL build output files (path defined via --player-save-path argument) {% endfor -%} {% endfor -%} {% endfor -%} From d16e93efb2835d602afeec779e969eb57c8ab9ce Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Mon, 3 Mar 2025 14:29:12 +0100 Subject: [PATCH 07/39] webgl correction --- .yamato/webgl-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/webgl-build.yml b/.yamato/webgl-build.yml index 507dffd349..53c61d6ac2 100644 --- a/.yamato/webgl-build.yml +++ b/.yamato/webgl-build.yml @@ -35,7 +35,7 @@ webgl_build_{{ project.name }}_{{ platform }}_{{ editor }}: image: {{ platform.image }} # Bokken image specification flavor: {{ platform.flavor }} # Memory/CPU allocation configuration commands: - - unity-downloader-cli -u {{ editor }} -c Editor -c webgl -c il2cpp # unity-downloader-cli should be already preinstalled on the image. This steps downloads proper editor to build from together with required components (webgl + il2cpp). + - unity-downloader-cli -u {{ editor }} -c Editor -c webgl -c il2cpp --wait # unity-downloader-cli should be already preinstalled on the image. This steps downloads proper editor to build from together with required components (webgl + il2cpp). The --wait flag will ensure that we wait until editor is downloaded # The following step builds the player with defined options such as: # Suite parameter if defined since it's a mandatory field to define which test suite should be used, but it doesn't matter in this case since we won't run any tests (--suite) From b57004dbd3e23dc591ab09743b528fd2a9ef9e97 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Mon, 3 Mar 2025 14:46:26 +0100 Subject: [PATCH 08/39] project-pack comments --- .yamato/project-pack.yml | 46 ++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml index 0963d99271..87517b1589 100644 --- a/.yamato/project-pack.yml +++ b/.yamato/project-pack.yml @@ -1,23 +1,41 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Jobs that iterate through and packs all NGO projects listed (to use in different jobs) -# For this job no specific platform support and no running Unity instance is required so small agent (as per project.metafile definition) coul be used to save resources and speed up the process -# If everyone adheres to this rule it can create bottlenecks (since everyone would use this machine) so we decided to pack project with the same platform as the given job runs on -{% for project in projects.all -%} -{% for platform in test_platforms.desktop -%} -project_pack_-_{{ project.name }}_{{ platform.name }}: - name: Project Pack - {{ project.name }} [{{ platform.name }}] +# DESCRIPTION-------------------------------------------------------------------------- + # Defines project packaging jobs for NGO projects + # These jobs prepare project packages that will be consumed by other pipeline jobs + # While these jobs don't require Unity Editor, they're distributed across platforms to prevent bottlenecks that would occur if all packaging was done on minimal VMs + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated using nested loops through: + # 1. All NGO projects (from project.metafile) + # 2. Desktop platforms (Windows, Ubuntu, macOS) + # This allows even distribution of packaging workload across different platforms + # Each configuration creates a separate packaging job for better parallelization + +# TECHNICAL CONSTRAINTS---------------------------------------------------------------- + # - No Unity Editor instance required + # - No specific platform dependencies + # - Could use minimal VM resources but distributed for better throughput + # - Requires npm and upm-ci-utils package + # - Generates artifacts in upm-ci~/packages directory + + +#-------------------------------------------------------------------------------------- + +{% for project in projects.all -%} # Jobs are being defined per each project as in project.metafile (testproject, minimalproject, testproject-tools-integration). +{% for platform in test_platforms.desktop -%} # Jobs are being defined per each desktop platform as in project.metafile declaration (Win, Ubuntu, Mac) +project_pack_-_{{ project.name }}_{{ platform.name }}: # This is a job name when querying from code (for example in _run-all.yml). Notice that parts of it will be filled in from for loops. For example there will be a configuration "project_pack_-_testproject_win" and when querying for this job we could for example change some values to get a given job (for example change platform value) and this will query different job configuration. + name: Project Pack - {{ project.name }} [{{ platform.name }}] # This is a job name visible in Yamato (notice that here we also use "generated" fields from for loops agent: - type: {{ platform.type }} - image: {{ platform.image }} - flavor: {{ platform.flavor }} + type: {{ platform.type }} # Defines type from platform configuration (for example Unity::VM). For packing no GPU extension is needed + image: {{ platform.image }} # Bokken image specification + flavor: {{ platform.flavor }} # Memory/CPU allocation configuration commands: - - npm install -g upm-ci-utils@stable --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - - upm-ci project pack --project-path {{ project.path }} + - upm-ci project pack --project-path {{ project.path }} # upm-ci version of packing command (will be replaced by upm-pvp in the future) artifacts: packages: paths: - - "upm-ci~/packages/**/*" + - "upm-ci~/packages/**/*" # Generates artifacts in upm-ci~/packages directory {% endfor -%} {% endfor -%} \ No newline at end of file From a58e4f58203f281dfce6912fc30084adf55607b9 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Mon, 3 Mar 2025 14:57:21 +0100 Subject: [PATCH 09/39] Modified code coverage option name --- .yamato/code-coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 5838bb8a6e..1e0132ffe0 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -16,7 +16,7 @@ code_coverage_{{ platform.name }}_{{ editor }}: commands: - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage --coverage-results-path=upm-ci~/test-results/CodeCoverage --code-coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage --coverage-results-path=upm-ci~/test-results/CodeCoverage --coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results artifacts: logs: paths: From e772cd72c46c27ecada8820ab4078689dd4630c0 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Mon, 3 Mar 2025 15:08:55 +0100 Subject: [PATCH 10/39] project-updated-dependencies comments --- .yamato/project-updated-dependencies-test.yml | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/.yamato/project-updated-dependencies-test.yml b/.yamato/project-updated-dependencies-test.yml index 513087f3c4..b06b23623a 100644 --- a/.yamato/project-updated-dependencies-test.yml +++ b/.yamato/project-updated-dependencies-test.yml @@ -1,27 +1,45 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Executes package tests for each package referenced in the project using the newest compatible version of the package dependencies. -# This is just to ensure that no critical change is made. It will be run on a default editor and default platform (it can be discussed if we should run it per all editors and platforms) -{% for project in projects.default -%} -{% for platform in test_platforms.default -%} -{% for editor in validation_editors.default -%} +# DESCRIPTION-------------------------------------------------------------------------- + # Validates package compatibility with latest dependency versions + # Tests each package referenced in the project against newest compatible dependency versions + # Helps detect potential breaking changes from dependency updates early + # Runs on default editor/platform to balance between coverage and resource usage + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated using nested loops through: + # 1. Default projects only (optimized subset from project.metafile) + # 2. Default platform (typically Ubuntu for resource efficiency) + # 3. Default editor version (latest stable) + # This focused configuration allows quick validation while managing resource usage + +# TECHNICAL CONSTRAINTS---------------------------------------------------------------- + # - Requires successful project packaging before execution + # - Tests only NGO package dependencies (com.unity.netcode.gameobjects) + # - Uses default configurations to optimize CI resource usage + # - Generates test results in upm-ci~/test-results directory + # - Dependencies must follow semantic versioning for proper version resolution + +#-------------------------------------------------------------------------------------- + +{% for project in projects.all -%} # Jobs are being defined per each project as in project.metafile (testproject, minimalproject, testproject-tools-integration). +{% for platform in test_platforms.desktop -%} # Jobs are being defined per each desktop platform as in project.metafile declaration (Win, Ubuntu, Mac) +{% for editor in validation_editors.all -%} # Jobs are being defined per each supported editor (For NGOv2.X it means 6000+) updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: name : Updated Dependencies Test - NGO {{ project.name }} [{{ platform.name }}, {{ editor }}] agent: - type: {{ platform.type }} - image: {{ platform.image }} - flavor: {{ platform.flavor }} + type: {{ platform.type }} # Defines type from platform configuration (for example Unity::VM). For packing no GPU extension is needed + image: {{ platform.image }} # Bokken image specification + flavor: {{ platform.flavor }} # Memory/CPU allocation configuration commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type updated-dependencies-tests --package-filter com.unity.netcode.gameobjects + - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type updated-dependencies-tests --package-filter com.unity.netcode.gameobjects # Execute dependency update tests with specific package filter artifacts: logs: paths: - - "upm-ci~/test-results/**/*" + - "upm-ci~/test-results/**/*" # Test execution logs and results dependencies: - - path: .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} - rerun: always + - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} # Requires packaged project to be available {% endfor -%} {% endfor -%} {% endfor -%} From 530c379ce83bff64b9a8c4ebc991646a0894a179 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Mon, 3 Mar 2025 15:30:26 +0100 Subject: [PATCH 11/39] Comments pass 1 --- .yamato/_run-all.yml | 41 ++++++++++++++++++++++++++- .yamato/_triggers.yml | 29 ++++++++++++++++++- .yamato/console-standalone-test.yml | 42 +++++++++++++++++++++------- .yamato/desktop-standalone-tests.yml | 36 ++++++++++++++++++------ .yamato/mobile-standalone-test.yml | 31 +++++++++++++------- .yamato/package-pack.yml | 30 +++++++++++++++++--- .yamato/package-tests.yml | 32 ++++++++++++++++++--- .yamato/performance-tests.yml | 31 ++++++++++++++++++-- .yamato/project-standards.yml | 34 ++++++++++++++++++---- .yamato/project-tests.yml | 35 ++++++++++++++++++----- .yamato/project.metafile | 10 ++++++- 11 files changed, 296 insertions(+), 55 deletions(-) diff --git a/.yamato/_run-all.yml b/.yamato/_run-all.yml index af4c78b7f0..86194e8ddc 100644 --- a/.yamato/_run-all.yml +++ b/.yamato/_run-all.yml @@ -1,6 +1,45 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- +# DESCRIPTION-------------------------------------------------------------------------- +# Defines aggregation jobs that group related test executions +# Provides both full validation (all editors) and trunk-only variants +# Groups tests by type (package, project) and platform (desktop, mobile, console) +# Enables efficient test execution for different validation scenarios + +# AGGREGATION PATTERNS--------------------------------------------------------------- +# Two main patterns for each test type: +# 1. Complete Validation: All supported Unity Editor versions +# 2. Trunk Validation: Only latest Unity Editor version +# +# Test Types: +# - Package Tests: NGO package validation +# - Project Tests: Project-specific validation +# - Standards Tests: Code standards verification +# - Platform Tests: Platform-specific builds and tests +# * Desktop (Windows, macOS, Linux) +# * Mobile (iOS, Android) +# * Console (PS4, PS5, Switch, Xbox) +# * WebGL (build-only validation) + +# USAGE GUIDELINES------------------------------------------------------------------ +# Complete Validation: +# - Use for release validation +# - Weekly automated runs +# - Major version updates +# +# Trunk Validation: +# - Use for quick verification +# - Daily automated runs +# - PR validation +# +# Platform-Specific: +# - Use when targeting specific platform changes +# - Hardware-dependent validation +# - Platform certification requirements + +#----------------------------------------------------------------------------------- + # Runs all package tests run_all_package_tests: name: Run All Package Tests diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index 8abd3622cf..adfb5b4a33 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -1,6 +1,33 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- +# DESCRIPTION-------------------------------------------------------------------------- +# Defines three main CI trigger patterns: +# 1. Pull Request Validation: Basic validation for PR changes +# 2. Nightly Development: Comprehensive testing on trunk +# 3. Weekly Full Validation: Complete testing across all editor versions +# Each pattern represents different balance between validation depth and execution time + +# TRIGGER PATTERNS------------------------------------------------------------------- +# Pull Request: +# - Triggers on PRs to develop, develop-2.0.0, and release branches +# - Focuses on critical validation paths +# - Cancels previous runs on new commits +# - Excludes draft PRs +# +# Nightly: +# - Runs daily on develop-2.0.0 +# - Includes all test types but only on trunk +# - Adds platform-specific and APV validation +# +# Weekly: +# - Most comprehensive validation +# - Runs across all supported editor versions +# - Includes code coverage analysis +# - Validates all projects and standards + +#----------------------------------------------------------------------------------- + # Run all relevant tasks when a pull request targeting the develop or release branch is created or updated. pull_request_trigger: name: Pull Request Trigger (develop, develop-2.0.0, & release branches) diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index 4ab96d8273..e9ba8ef6b1 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -1,17 +1,39 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Builds a player on console standalone platform and executes RuntimeTests (equivalent to PlayMode tests) of the NGO package in the Standalone build. -# Default project (testptoject) in this case is used as a context. -# Builds/Tests are made on each console platform (PS4, PS5, Switch, XboxOne, XboxSeriesX) as in project.metafile declaration -# Builds/Tests are made on each supported editor as in project.metafile declaration +# DESCRIPTION-------------------------------------------------------------------------- + # Console platform validation for NGO package using standalone builds + # Executes runtime tests (PlayMode equivalent) in standalone console builds + # Uses split build/run approach due to console-specific requirements + # Covers PS4, PS5, Switch, XboxOne, and XboxSeriesX platforms -# For SOME of the console devices it's necessary to split build and run phases so it was split for all -# For all consoles we need to use il2cpp scripting backend (so no testing with mono) -# Switch works only with ARM64 and the rest with x64 architectures -# For now all platforms used for building are windows based + # PLATFORM SPECIFICS----------------------------------------------------------------- + # Common Requirements: + # - All consoles require IL2CPP scripting backend + # - Windows-based build machines required + # - Platform-specific SDK environment variables + # + # Platform-Specific: + # - Switch: ARM64 architecture only + # - Other Consoles: x64 architecture + # - Each console requires specific SDK paths and tools + # - Some consoles require specific hardware models for testing -# Builds are made with x64 architecture machines since those are compatible to run on ARM64 devices + # TECHNICAL CONSTRAINTS-------------------------------------------------------------- + # Build Phase: + # - Windows build agents required + # - Platform-specific SDK installations + # - IL2CPP only (no Mono support) + # - Architecture varies by console + # + # Run Phase: + # - Requires specific console hardware + # - Platform-specific environment setup + # - Direct hardware access needed + # - Specific SDK versions required + +#----------------------------------------------------------------------------------- + {% for project in projects.default -%} {% for platform in test_platforms.console_build -%} {% for editor in validation_editors.all -%} diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 4d8fc8167f..89cf736f34 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -1,14 +1,32 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Builds a player on desktop standalone platform and executes RuntimeTests (equivalent to PlayMode tests) of the NGO package in the Standalone build. -# Default project (testptoject) in this case is used as a context. -# Builds are made on each desktop platform as in project.metafile declaration -# Builds are made on each supported editor as in project.metafile declaration -# Builds are made with different scripting backends as in project.metafile declaration -# ARM64 architectures are for now not considered since Windows_arm64 is recommended to use only after builds (would require separation here) and when it comes to macOS_arm64 there is problem with OpenCL not being available +# DESCRIPTION-------------------------------------------------------------------------- + # Desktop platform validation for NGO package using standalone builds + # Executes runtime tests (PlayMode equivalent) in standalone player builds + # Uses split build and run approach for better resource utilization + # Covers all supported desktop platforms with various configurations -# Build phase +# CONFIGURATION MATRIX--------------------------------------------------------------- + # Tests are executed across multiple dimensions: + # 1. Desktop Platforms: Windows, macOS, Linux + # 2. Unity Editor Versions: All supported versions + # 3. Scripting Backends: IL2CPP and Mono + # Note: ARM64 architectures are currently excluded because: + # - Windows ARM64 is post-build only + # - macOS ARM64 has OpenCL availability issues + +# TECHNICAL CONSTRAINTS-------------------------------------------------------------- + # - GPU requirement varies by platform (required for Windows/Linux, optional for macOS) + # - Default project used as test context + # - Null graphics API used for test execution + # - Separate build and run phases for better resource management + # - 30-minute timeout for both build and run phases + +#----------------------------------------------------------------------------------- + + +# Build phase-------------------------------------------------------------------------- {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} @@ -43,7 +61,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ -# Run phase +# Run phase-------------------------------------------------------------------------------- {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index e8ac07823a..600cd8155b 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -1,16 +1,27 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- + +# DESCRIPTION-------------------------------------------------------------------------- + # Mobile platform validation for NGO package consisting of two phases: + # 1. Build Phase: Creates standalone players for mobile platforms + # 2. Run Phase: Executes runtime tests on actual mobile devices + # Uses default project as test context for consistency + # Covers both iOS and Android platforms with platform-specific requirements + +# PLATFORM SPECIFICS----------------------------------------------------------------- + # iOS Requirements: + # - Must use IL2CPP scripting backend + # - Only supports ARM64 architecture + # - Builds on macOS agents only + # + # Android Requirements: + # - Uses IL2CPP scripting backend (recommended over Mono) + # - Supports both ARM64 and ARMv7 architectures + # - Builds on any desktop platform + # + # Note: All builds use x64 machines for compatibility with ARM64 target devices -# Builds a player on mobile standalone platform and executes RuntimeTests (equivalent to PlayMode tests) of the NGO package in the Standalone build. -# Default project (testptoject) in this case is used as a context. -# Builds/Tests are made on each mobile platform (Android and iOS) as in project.metafile declaration -# Builds/Tests are made on each supported editor as in project.metafile declaration - -# For mobile devices it's necessary to split build and run phases -# For iOS we need to use il2cpp. For android we could use both but il2cpp is recommended so for now we will only use il2cpp as scripting backend -# iOS works only with ARM64 and Android is tested with both ARM64 and ARMv7 -# Builds are made with x64 architecture machines since those are compatible to run on ARM64 devices {% for project in projects.default -%} {% for platform in test_platforms.mobile_build -%} {% for editor in validation_editors.all -%} diff --git a/.yamato/package-pack.yml b/.yamato/package-pack.yml index a9ab902847..3435dc1b9f 100644 --- a/.yamato/package-pack.yml +++ b/.yamato/package-pack.yml @@ -1,9 +1,31 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Packs Netcode for GameObjects together with performing initial checks. -# For this job no specific platform support and no running Unity instance is required so small agent (as per project.metafile definition) could be used to save resources and speed up the process. -# If everyone adheres to this rule it can create bottlenecks (since everyone would use this machine) so we decided to pack with the same platform as the given job runs on +# DESCRIPTION-------------------------------------------------------------------------- + # Packages Netcode for GameObjects (NGO) and performs initial validation checks + # Uses Package Verification Pipeline (PVP) for both packaging and validation + # Includes x-ray validation for quick package structure verification + # Distributed across platforms to prevent resource bottlenecks + # Note: This is the first step in the package validation pipeline + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated for each desktop platform: + # - Windows, Ubuntu, macOS + # While packaging doesn't require specific platforms, distribution helps: + # - Prevent bottlenecks on minimal VMs + # - Balance workload across available resources + # - Maintain consistent environment with dependent jobs + +# TECHNICAL CONSTRAINTS--------------------------------------------------------------- + # - No Unity Editor instance required + # - Quick execution (15 minute timeout) + # - Supports custom validation exceptions via pvpExceptions.json + # - Uses PVP x-ray for lightweight validation + # - Generates both package artifacts and validation results + # - Allows missing checks (--allow-missing) for x-ray phase + +#------------------------------------------------------------------------------------ + {% for platform in test_platforms.desktop -%} package_pack_-_ngo_{{ platform.name }}: name: Package Pack (and x-ray) - NGO [{{ platform.name }}] diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index c4172797f8..52d23549c0 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -1,9 +1,33 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Executes PlayMode and EditMode tests of the NGO package in the Editor context -# Those tests run in the editor so we don't need to consider different scripting backends or architectures -# Tests are executed for all supported editors on each desktop platform as in project.metafile declaration +# DESCRIPTION-------------------------------------------------------------------------- + # Executes NGO package validation and testing in Editor context + # Combines Package Verification Pipeline (PVP) validation with UTR testing + # Validates package structure, API compatibility, and runtime behavior + # Covers both PlayMode and EditMode tests for the package + # Note: Performance tests are explicitly excluded from this validation + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated using nested loops through: + # 1. All desktop platforms (Windows, Ubuntu, macOS) + # 2. All supported Unity Editor versions + # Package testing is comprehensive across environments because: + # - Different platforms may expose platform-specific issues + # - Editor versions may have different compatibility requirements + # - API compatibility needs validation across all supported versions + +# TECHNICAL CONSTRAINTS--------------------------------------------------------------- + # - Runs in Editor context only (no player builds) + # - No scripting backend variations needed + # - Requires successful package pack job + # - Uses PVP for package validation + # - Requires specific display configuration for Ubuntu + # - Supports custom validation profiles via pvpExceptions.json + # - Extended logging enabled via UNITY_EXT_LOGGING + +#------------------------------------------------------------------------------------ + {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} package_test_-_ngo_{{ editor }}_{{ platform.name }}: diff --git a/.yamato/performance-tests.yml b/.yamato/performance-tests.yml index cecf3aed95..66e37de536 100644 --- a/.yamato/performance-tests.yml +++ b/.yamato/performance-tests.yml @@ -1,8 +1,33 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Performance tests for the `com.unity.netcode.gameobjects` package. No performance data will be reported. -# Performance tests are executed within editor on desktop platforms in context of default project (testproject) +# DESCRIPTION-------------------------------------------------------------------------- + # Executes performance tests for NGO package without data reporting + # Tests run in Editor context using default project configuration + # Focuses on validating performance test execution rather than metrics collection + # Serves as a validation step to ensure performance tests are functioning correctly + # Note: This configuration explicitly disables performance data reporting + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated using nested loops through: + # 1. All desktop platforms (Windows, Ubuntu, macOS) + # 2. All supported Unity Editor versions + # 3. Default project only (testproject) + # Comprehensive platform/editor coverage ensures test stability + # Uses default project to maintain consistent test environment + +# TECHNICAL CONSTRAINTS--------------------------------------------------------------- + # - Requires Unity Editor installation + # - Tests run in Editor context only + # - 3600 second (1 hour) timeout limit + # - Focuses on Unity.NetCode.* assemblies + # - Performance category tests only + # - Disables package manager traces + # - Includes extensive log collection + # - No performance metrics are reported to monitoring systems + +#------------------------------------------------------------------------------------ + {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} {% for project in projects.default -%} diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index b0a74ad6a6..53f2c6c331 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -1,9 +1,33 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Project standards are being checked for package (in project context). -# It should be enough to perform the test on default project (testproject) but to make sure that all projects conform to our standards job for each of them will be available. -# Tests are executed for default editor (trunk) on default platform (ubuntu) as in project.metafile declaration since results would be the same across editors and platforms. +# DESCRIPTION-------------------------------------------------------------------------- + # Validates project compliance with NGO coding standards and conventions + # While testproject validation would be sufficient, all projects are checked + # to ensure consistent code quality across the entire codebase + # Standards validation includes: + # - Code formatting compliance + # - Project structure validation + # - Coding convention adherence + # - Solution file synchronization + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated using nested loops through: + # 1. All projects (comprehensive validation) + # 2. Default platform only (Ubuntu for efficiency) + # 3. Default editor version (trunk) + # Results are consistent across platforms/editors, so default configurations + # are used to optimize CI resource usage + +# TECHNICAL CONSTRAINTS--------------------------------------------------------------- + # - Requires .NET SDK installation + # - Needs Unity Editor for solution synchronization + # - Uses custom standards validation tool (netcode.standards) + # - Platform/editor independent validation + # - Generates no test artifacts (pass/fail only) + +#------------------------------------------------------------------------------------ + {% for project in projects.all -%} {% for platform in test_platforms.default -%} {% for editor in validation_editors.default -%} @@ -14,7 +38,7 @@ standards_{{ platform.name }}_{{ project.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - # .NET environment setup + # .NET environment setup. Ensures required .NET SDK and formatting tools are available - dotnet --version - dotnet format --version diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index ec91e04540..0bc2c6cdb8 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -1,10 +1,31 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Executes PlayMode and EditMode tests of the given project (ones that have tests) in the Editor context -# Those tests don't include NGO package tests since they are handled on their respective jobs. -# Those tests run in the editor so we don't need to consider different scripting backends or architectures -# Tests are executed for all supported editors on each desktop platform as in project.metafile declaration +# DESCRIPTION-------------------------------------------------------------------------- + # Executes project-specific tests in Unity Editor context + # Covers both PlayMode and EditMode tests from project test assemblies + # Excludes NGO package tests (handled in separate package test jobs) + # Provides validation across all supported Unity Editor versions + # Note: These tests focus on project-specific functionality, not package validation + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated using nested loops through: + # 1. All projects with tests (filtered by has_tests flag) + # 2. All desktop platforms (Windows, Ubuntu, macOS) + # 3. All supported Unity Editor versions + # This ensures comprehensive testing across supported environments + # Each combination creates a separate test job for parallel execution + +# TECHNICAL CONSTRAINTS--------------------------------------------------------------- + # - Runs in Editor context only (no player builds required) + # - No scripting backend variations needed (Editor uses Mono) + # - Architecture variations not applicable (Editor context) + # - Requires project packaging as prerequisite + # - Special handling for Ubuntu display configuration + # - Excludes performance tests for consistent CI execution + +#------------------------------------------------------------------------------------ + {% for project in projects.all -%} {% if project.has_tests == "true" -%} {% for platform in test_platforms.desktop -%} @@ -22,9 +43,9 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: artifacts: logs: paths: - - "upm-ci~/test-results/**/*" + - "upm-ci~/test-results/**/*" # Test execution logs and results dependencies: - - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} # Requires packaged project to be available {% endfor -%} {% endfor -%} {% endif -%} diff --git a/.yamato/project.metafile b/.yamato/project.metafile index 43c1202c90..ff48f37fdb 100644 --- a/.yamato/project.metafile +++ b/.yamato/project.metafile @@ -1,3 +1,10 @@ +# DESCRIPTION-------------------------------------------------------------------------- +# Central configuration file defining test environments, platforms, and projects +# Used to generate CI pipeline configurations through templating +# Provides consistent environment definitions across all CI jobs +# Optimizes resource usage through appropriate agent selection + +# AGENT CONFIGURATIONS--------------------------------------------------------------- # The small agent was created to handle jobs that don't involve running Unity (for example pack job). # It uses ubuntu since Linux VMs are faster and cheaper to provision than Mac or Windows Virtual Machines (VMs). # The b1.small flavour is nearly always sufficient for jobs that don’t involve running Unity Editor. @@ -7,6 +14,7 @@ small_agent_platform: image: package-ci/ubuntu-22.04:v4 flavor: b1.small +# PLATFORM CONFIGURATIONS----------------------------------------------------------- # Platforms that project will/can be tested on. test_platforms: # Default platform is used for all basic jobs. Ubuntu was chosen since it's fastest and most available @@ -121,7 +129,7 @@ test_platforms: flavor: b1.large standalone: GameCoreScarlett - +# EDITOR CONFIGURATIONS------------------------------------------------------------ # Editors to be used for testing. # Since NGOv2 official support started from U6 it means that only those editors should be used for testing validation_editors: From 9d89d3abd4b33aac794ce033d4d2807b0ad230e1 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Tue, 4 Mar 2025 11:41:21 +0100 Subject: [PATCH 12/39] Coverage options --- .yamato/code-coverage.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 1e0132ffe0..70419f9acb 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -2,7 +2,7 @@ --- # Runs package tests in order to determine code coverage of the NGO package. -# In essence it's performing the same task as .yamato/package-test jobs with the overhead being the measured code coverage +# In essence, it's performing the same task as .yamato/package-test jobs with the overhead being the measured code coverage # It's ok for code coverage to be performed only on one platform (default) since code coverage won't change much between those. # Default platform was chosen (ubuntu) since it's the fastest and most resource friendly with default editor. {% for platform in test_platforms.default -%} @@ -16,13 +16,12 @@ code_coverage_{{ platform.name }}_{{ editor }}: commands: - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage --coverage-results-path=upm-ci~/test-results/CodeCoverage --coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage --coverage-results-path=test-results/CodeCoverage --coverage-options="generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results artifacts: logs: paths: - - "upm-ci~/test-results/CodeCoverage/**/*" - - "test-results/CoverageResults/**/*" - - "test-results/**/coverage.xml" + - "test-results/**/*" + - "test-results/CodeCoverage/**/*" dependencies: - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} From 75615dbbb4359741a6b8d5711877c71c37692f50 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Tue, 4 Mar 2025 13:56:30 +0100 Subject: [PATCH 13/39] Coverage local path specified --- .yamato/code-coverage.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 70419f9acb..509271af3d 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -16,12 +16,11 @@ code_coverage_{{ platform.name }}_{{ editor }}: commands: - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage --coverage-results-path=test-results/CodeCoverage --coverage-options="generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage -coverage-results-path=$PWD/test-results/CodeCoverage --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results artifacts: logs: paths: - "test-results/**/*" - - "test-results/CodeCoverage/**/*" dependencies: - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} From 84092cf58923a0d633cefd633eac75f1f42600e3 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Tue, 4 Mar 2025 16:44:44 +0100 Subject: [PATCH 14/39] Comments --- .yamato/_run-all.yml | 49 ++---------- .yamato/_triggers.yml | 40 +++++----- .yamato/code-coverage.yml | 40 ++++++++-- .yamato/console-standalone-test.yml | 80 +++++++++---------- .yamato/desktop-standalone-tests.yml | 54 ++++++------- .yamato/mobile-standalone-test.yml | 76 ++++++++++-------- .yamato/package-pack.yml | 30 +++---- .yamato/package-tests.yml | 43 +++++----- .yamato/performance-tests.yml | 50 +++++------- .yamato/project-pack.yml | 50 +++++++----- .yamato/project-standards.yml | 38 +++++---- .yamato/project-tests.yml | 44 +++++----- .yamato/project-updated-dependencies-test.yml | 49 +++++++----- .yamato/project.metafile | 47 +++++++---- .yamato/webgl-build.yml | 56 +++++++------ 15 files changed, 378 insertions(+), 368 deletions(-) diff --git a/.yamato/_run-all.yml b/.yamato/_run-all.yml index 86194e8ddc..f031dc7ff7 100644 --- a/.yamato/_run-all.yml +++ b/.yamato/_run-all.yml @@ -2,41 +2,8 @@ --- # DESCRIPTION-------------------------------------------------------------------------- -# Defines aggregation jobs that group related test executions -# Provides both full validation (all editors) and trunk-only variants -# Groups tests by type (package, project) and platform (desktop, mobile, console) -# Enables efficient test execution for different validation scenarios - -# AGGREGATION PATTERNS--------------------------------------------------------------- -# Two main patterns for each test type: -# 1. Complete Validation: All supported Unity Editor versions -# 2. Trunk Validation: Only latest Unity Editor version -# -# Test Types: -# - Package Tests: NGO package validation -# - Project Tests: Project-specific validation -# - Standards Tests: Code standards verification -# - Platform Tests: Platform-specific builds and tests -# * Desktop (Windows, macOS, Linux) -# * Mobile (iOS, Android) -# * Console (PS4, PS5, Switch, Xbox) -# * WebGL (build-only validation) - -# USAGE GUIDELINES------------------------------------------------------------------ -# Complete Validation: -# - Use for release validation -# - Weekly automated runs -# - Major version updates -# -# Trunk Validation: -# - Use for quick verification -# - Daily automated runs -# - PR validation -# -# Platform-Specific: -# - Use when targeting specific platform changes -# - Hardware-dependent validation -# - Platform certification requirements + # Those jobs group together related test executions + # Enables efficient test execution for different validation scenarios #----------------------------------------------------------------------------------- @@ -112,7 +79,7 @@ run_all_webgl_builds: {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} - - .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform }}_{{ editor }} + - .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} @@ -125,7 +92,7 @@ run_all_webgl_builds_trunk: {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.default -%} - - .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform }}_{{ editor }} + - .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} @@ -167,7 +134,7 @@ run_all_project_tests_mobile_standalone: {% for project in projects.default -%} {% for platform in test_platforms.mobile_test -%} {% for editor in validation_editors.all -%} - - .yamato/mobile-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} + - .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} @@ -180,7 +147,7 @@ run_all_project_tests_mobile_standalone_trunk: {% for project in projects.default -%} {% for platform in test_platforms.mobile_test -%} {% for editor in validation_editors.default -%} - - .yamato/mobile-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} + - .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} @@ -193,7 +160,7 @@ run_all_project_tests_console_standalone: {% for project in projects.default -%} {% for platform in test_platforms.console_test -%} {% for editor in validation_editors.all -%} - - .yamato/console-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} + - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} @@ -206,7 +173,7 @@ run_all_project_tests_console_standalone_trunk: {% for project in projects.default -%} {% for platform in test_platforms.console_test -%} {% for editor in validation_editors.default -%} - - .yamato/console-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} + - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index adfb5b4a33..89d8149ad1 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -3,28 +3,28 @@ # DESCRIPTION-------------------------------------------------------------------------- # Defines three main CI trigger patterns: -# 1. Pull Request Validation: Basic validation for PR changes -# 2. Nightly Development: Comprehensive testing on trunk -# 3. Weekly Full Validation: Complete testing across all editor versions -# Each pattern represents different balance between validation depth and execution time + # 1. Pull Request Validation: Basic validation for PR changes + # 2. Nightly Development: Comprehensive testing on trunk + # 3. Weekly Full Validation: Complete testing across all editor versions +# Each pattern represents different balance between validation depth, execution time and CI resource usage # TRIGGER PATTERNS------------------------------------------------------------------- -# Pull Request: -# - Triggers on PRs to develop, develop-2.0.0, and release branches -# - Focuses on critical validation paths -# - Cancels previous runs on new commits -# - Excludes draft PRs -# -# Nightly: -# - Runs daily on develop-2.0.0 -# - Includes all test types but only on trunk -# - Adds platform-specific and APV validation -# -# Weekly: -# - Most comprehensive validation -# - Runs across all supported editor versions -# - Includes code coverage analysis -# - Validates all projects and standards + # Pull Request: + # Triggers on PRs to develop, develop-2.0.0, and release branches + # Focuses on critical validation paths that we should validate before merging PRs + # Cancels previous runs on new commits + # Excludes draft PRs + + # Nightly: + # Runs daily on develop-2.0.0 (local configuration) + # Includes all test types but only on trunk. TODO: Add validation for minimum supported editor in nightly + # Adds platform-specific and APV validation + + # Weekly: + # Most comprehensive validation + # Runs across all supported editor versions + # Includes code coverage analysis + # Validates all projects and standards #----------------------------------------------------------------------------------- diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 509271af3d..3fd029adae 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -1,10 +1,36 @@ {% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- - -# Runs package tests in order to determine code coverage of the NGO package. -# In essence, it's performing the same task as .yamato/package-test jobs with the overhead being the measured code coverage -# It's ok for code coverage to be performed only on one platform (default) since code coverage won't change much between those. -# Default platform was chosen (ubuntu) since it's the fastest and most resource friendly with default editor. + +# DESCRIPTION-------------------------------------------------------------------------- + # This job executes package tests with code coverage analysis enabled for the NGO (Netcode for GameObjects) package + # Coverage analysis provides insights into: + # Test coverage metrics for NGO assemblies + # Line and branch coverage statistics + # Generated HTML reports for coverage visualization + # Additional metrics for coverage analysis + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # Jobs are generated using nested loops through: + # 1. For default platform only (Ubuntu) since coverage would not vary between platforms (no need for checks on more platforms) + # 2. For default editor version (trunk) since coverage would not vary between editors (no need for checks on more editors) + + # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. + # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. + + # The jobs name being generated would be for example "Code Coverage - NGO [testproject, 6000.0]" (name visible in Yamato) or "code_coverage_testproject_6000.0" (job name when querying for it) + +#TECHNICAL CONSTRAINTS--------------------------------------------------------------- + # Requires Unity Editor installation + # Uses UnifiedTestRunner (UTR) for test execution + # Depends on successful package pack job completion + # Burst compilation is disabled to ensure accurate coverage measurement + +# QUALITY THOUGHTS-------------------------------------------------------------------- + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + + + {% for platform in test_platforms.default -%} {% for editor in validation_editors.default -%} code_coverage_{{ platform.name }}_{{ editor }}: @@ -14,9 +40,9 @@ code_coverage_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait + - unity-downloader-cli -u {{ editor }} -c Editor - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage -coverage-results-path=$PWD/test-results/CodeCoverage --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage -coverage-results-path=$PWD/test-results/CodeCoverage --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results artifacts: logs: paths: diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index e9ba8ef6b1..b6027e1efc 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -2,55 +2,54 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # Console platform validation for NGO package using standalone builds - # Executes runtime tests (PlayMode equivalent) in standalone console builds - # Uses split build/run approach due to console-specific requirements - # Covers PS4, PS5, Switch, XboxOne, and XboxSeriesX platforms + # This file contains Console platform test validation for NGO package consisting of two phases: + # 1. Build Phase: Creates standalone players for console platforms + # 2. Run Phase: Executes runtime tests on actual console devices + # Those phases need to be separated (in contrast to for example desktop tests) - # PLATFORM SPECIFICS----------------------------------------------------------------- +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: + # 1. For all console platform (Switch, ps4, ps5, xbox360, xboxOne) defined in test_platforms.console_ (in project.metafile) + # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors defined in project.metafile) + # 3. For default project which serves as context + + # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. + # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. + + # The jobs name being generated would be for example "Build testproject - [win, 6000.0, il2cpp]" (name visible in Yamato) or "console_standalone_build_testproject_win_6000.0" (job name when querying for it) + +# PLATFORM SPECIFICS----------------------------------------------------------------- # Common Requirements: - # - All consoles require IL2CPP scripting backend - # - Windows-based build machines required - # - Platform-specific SDK environment variables - # + # All consoles require IL2CPP scripting backend + # Platform-specific SDK environment variables + # Platform-Specific: - # - Switch: ARM64 architecture only - # - Other Consoles: x64 architecture - # - Each console requires specific SDK paths and tools - # - Some consoles require specific hardware models for testing - - # TECHNICAL CONSTRAINTS-------------------------------------------------------------- - # Build Phase: - # - Windows build agents required - # - Platform-specific SDK installations - # - IL2CPP only (no Mono support) - # - Architecture varies by console - # - # Run Phase: - # - Requires specific console hardware - # - Platform-specific environment setup - # - Direct hardware access needed - # - Specific SDK versions required + # Switch: ARM64 architecture only + # Other Consoles: x64 architecture + # Each console requires specific SDK paths and tools + + # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices + # Note: More of a Unity specific but test assemblies need to be included in the build phase command -#----------------------------------------------------------------------------------- +# QUALITY THOUGHTS-------------------------------------------------------------------- + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + + +# BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.console_build -%} {% for editor in validation_editors.all -%} -build_{{ project.name }}_{{ platform.name }}_{{ editor }}: +console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: name: Build {{ project.name }} - [{{ platform.name }}, {{ editor }}, il2cpp{% if platform.name == "switch" %}, arm64{% endif %}] agent: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - # Installing tools. unity-downloader-cli and utr should be already preinstalled on the image - - sudo pip install unity-downloader-cli - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --fast --wait - - # Platform specific Build + - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 - variables: # PS4 related SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00' @@ -74,11 +73,11 @@ build_{{ project.name }}_{{ platform.name }}_{{ editor }}: -# Executes RuntimeTests of the NGO package in the Standalone build for consoles +# RUN PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.console_test -%} {% for editor in validation_editors.all -%} -run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: +console_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: name: Run {{ project.name }} Tests - [{{ platform.name }}, {{ editor }}, il2cpp] agent: type: {{ platform.type }} @@ -88,13 +87,8 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: model: {{ platform.model }} {% endif %} commands: - # Installing tools. - - sudo pip install unity-downloader-cli - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --fast --wait - - # Platform specific Execution + - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 - variables: # PS4 related SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00' @@ -110,7 +104,7 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: paths: - "test-results/**/*" dependencies: - - .yamato/console-standalone-test.yml#build_{{ project.name }}_{{ platform.name }}_{{ editor }} + - .yamato/console-standalone-test.yml#console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} \ No newline at end of file diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 89cf736f34..a47a136e56 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -2,31 +2,31 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # Desktop platform validation for NGO package using standalone builds - # Executes runtime tests (PlayMode equivalent) in standalone player builds - # Uses split build and run approach for better resource utilization - # Covers all supported desktop platforms with various configurations + # This file contains Desktop platform test validation for NGO package consisting of two phases: + # 1. Build Phase: Creates standalone players for desktop platforms + # 2. Run Phase: Executes runtime tests on actual desktop devices + # For desktop devices the phase separation is not necessary, but we did it for consistency with other standalone tests configurations -# CONFIGURATION MATRIX--------------------------------------------------------------- - # Tests are executed across multiple dimensions: - # 1. Desktop Platforms: Windows, macOS, Linux - # 2. Unity Editor Versions: All supported versions - # 3. Scripting Backends: IL2CPP and Mono - # Note: ARM64 architectures are currently excluded because: - # - Windows ARM64 is post-build only - # - macOS ARM64 has OpenCL availability issues +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: + # 1. For all desktop platform (Windows, macOS, Ubuntu) defined in test_platforms.desktop_ (in project.metafile) + # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors defined in project.metafile) + # 3. For default project which serves as context + # 4. For all scripting backends -# TECHNICAL CONSTRAINTS-------------------------------------------------------------- - # - GPU requirement varies by platform (required for Windows/Linux, optional for macOS) - # - Default project used as test context - # - Null graphics API used for test execution - # - Separate build and run phases for better resource management - # - 30-minute timeout for both build and run phases + # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. + # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. + + # The jobs name being generated would be for example "Standalone Build - NGO testproject - [win, mono, 6000.0]" (name visible in Yamato) or "desktop_standalone_build_testproject_win_mono_6000.0" (job name when querying for it) + +# QUALITY THOUGHTS-------------------------------------------------------------------- + # To see where this job is included (in trigger job definitions) look into _triggers.yml file #----------------------------------------------------------------------------------- - -# Build phase-------------------------------------------------------------------------- + +# BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} @@ -38,10 +38,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} --fast --wait # Installing editor - -# Build Player + - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: @@ -50,7 +47,6 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ logs: paths: - "artifacts/**/*" - dependencies: - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} {% endfor -%} @@ -61,7 +57,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ -# Run phase-------------------------------------------------------------------------------- +# RUN PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} @@ -73,12 +69,8 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} --fast --wait - -# Run Standalone tests + - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 - artifacts: logs: paths: diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index 600cd8155b..09fde91637 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -2,38 +2,53 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # Mobile platform validation for NGO package consisting of two phases: - # 1. Build Phase: Creates standalone players for mobile platforms - # 2. Run Phase: Executes runtime tests on actual mobile devices - # Uses default project as test context for consistency - # Covers both iOS and Android platforms with platform-specific requirements + # This file contains Mobile platform test validation for NGO package consisting of two phases: + # 1. Build Phase: Creates standalone players for mobile platforms + # 2. Run Phase: Executes runtime tests on actual mobile devices + # Those phases need to be separated (in contrast to for example desktop tests) + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: + # 1. For all mobile platform (Android, iOS) defined in test_platforms.mobile_ (in project.metafile) + # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors defined in project.metafile) + # 3. For default project which serves as context + + # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. + # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. + + # The jobs name being generated would be for example "Build testproject - [win, 6000.0, il2cpp]" (name visible in Yamato) or "build_testproject_win_6000.0" (job name when querying for it) # PLATFORM SPECIFICS----------------------------------------------------------------- # iOS Requirements: - # - Must use IL2CPP scripting backend - # - Only supports ARM64 architecture - # - Builds on macOS agents only - # - # Android Requirements: - # - Uses IL2CPP scripting backend (recommended over Mono) - # - Supports both ARM64 and ARMv7 architectures - # - Builds on any desktop platform - # - # Note: All builds use x64 machines for compatibility with ARM64 target devices - + # Must use IL2CPP scripting backend + # Only supports ARM64 architecture + # Builds on macOS agents only + + # Android Requirements: + # Uses IL2CPP scripting backend (recommended over Mono) + # Supports both ARM64 and ARMv7 architectures + # Can be build on any desktop platform + # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices + # Note: More of a Unity specific but test assemblies need to be included in the build phase command + +# QUALITY THOUGHTS-------------------------------------------------------------------- + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + + +# BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.mobile_build -%} {% for editor in validation_editors.all -%} -build_{{ project.name }}_{{ platform.name }}_{{ editor }}: - name: Build {{ project.name }} - [{{ platform.name }}, {{ editor }}, il2cpp] +mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: + name: Build {{ project.name }} - [{{ platform.name }}, {{ editor }}, il2cpp] agent: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple # Installing unity-downloader-cli - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} --fast --wait # Installing editor + - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: @@ -43,20 +58,16 @@ build_{{ project.name }}_{{ platform.name }}_{{ editor }}: logs: paths: - "artifacts/**/*" - - variables: - CI: true - ENABLE_BURST_COMPILATION: False {% endfor -%} {% endfor -%} {% endfor -%} -# Executes RuntimeTests of the NGO package in the Standalone build for mobiles +# RUN PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.mobile_test -%} {% for editor in validation_editors.all -%} -run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: +mobile_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: name: Run {{ project.name }} Tests - [{{ platform.name }}, {{ editor }}, il2cpp] agent: type: {{ platform.type }} @@ -65,14 +76,9 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: {% if platform.model %} model: {{ platform.model }} {% endif %} - # Skip repository cloning - skip_checkout: true commands: - # Installing tools - - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - # Installing editor. We still need the editor to run tests on standalone build - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} --fast --wait + - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} {% if platform.standalone == "Android" %} # Download standalone UnityTestRunner and ADB setup @@ -83,13 +89,13 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: # Connect to Android device - command: '"%ANDROID_SDK_ROOT%\platform-tools\adb.exe" connect %BOKKEN_DEVICE_IP%' - # Run tests + # Run tests for Android devices - | set ANDROID_DEVICE_CONNECTION=%BOKKEN_DEVICE_IP% UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 {% else %} - # Run tests + # Run tests for non-Android devices - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 {% endif %} artifacts: @@ -97,7 +103,7 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: paths: - "test-results/**/*" dependencies: - - .yamato/mobile-standalone-test.yml#build_{{ project.name }}_{{ platform.name }}_{{ editor }} + - .yamato/mobile-standalone-test.yml#mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} diff --git a/.yamato/package-pack.yml b/.yamato/package-pack.yml index 3435dc1b9f..db47ddcca6 100644 --- a/.yamato/package-pack.yml +++ b/.yamato/package-pack.yml @@ -2,27 +2,23 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # Packages Netcode for GameObjects (NGO) and performs initial validation checks - # Uses Package Verification Pipeline (PVP) for both packaging and validation - # Includes x-ray validation for quick package structure verification - # Distributed across platforms to prevent resource bottlenecks - # Note: This is the first step in the package validation pipeline + # This job is responsible for packing a specific package. It generates package artifacts (.tgz) required for testing and publishing, ensuring all dependencies are properly bundled and validated before any test execution. + # The job itself doesn't test anything specific but rather it prepares project packages that will be consumed by other pipeline jobs. + # The job performs additional validation by using Package Verification Pipeline (PVP). It includes x-ray validation for quick package structure verification + # Because of that we can detect errors at the early stage of testing so not to waste CI resources # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # Jobs are generated for each desktop platform: - # - Windows, Ubuntu, macOS - # While packaging doesn't require specific platforms, distribution helps: - # - Prevent bottlenecks on minimal VMs - # - Balance workload across available resources - # - Maintain consistent environment with dependent jobs + # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # Jobs are generated using nested loops through: + # 1. For all desktop platform (Windows, Ubuntu, macOS) defined in test_platforms.desktop (in project.metafile) # TECHNICAL CONSTRAINTS--------------------------------------------------------------- - # - No Unity Editor instance required - # - Quick execution (15 minute timeout) - # - Supports custom validation exceptions via pvpExceptions.json - # - Uses PVP x-ray for lightweight validation - # - Generates both package artifacts and validation results - # - Allows missing checks (--allow-missing) for x-ray phase + # Job does not require Unity Editor in order to perform packing. + # Job uses PVP x-ray for lightweight validation + # Job generates both packages artifacts and pvp-results file. + +# QUALITY THOUGHTS-------------------------------------------------------------------- + # To see where this job is included (in trigger job definitions) look into _triggers.yml file #------------------------------------------------------------------------------------ diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index 52d23549c0..49b0044727 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -2,29 +2,31 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # Executes NGO package validation and testing in Editor context - # Combines Package Verification Pipeline (PVP) validation with UTR testing - # Validates package structure, API compatibility, and runtime behavior - # Covers both PlayMode and EditMode tests for the package - # Note: Performance tests are explicitly excluded from this validation + # This job executes package-specific tests in Unity Editor context + # Those tests cover both PlayMode and EditMode tests from package test assemblies + # Additionally it combines Package Verification Pipeline (PVP) validation. This ensures that package is compatible with Unity standards # CONFIGURATION STRUCTURE-------------------------------------------------------------- + # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: - # 1. All desktop platforms (Windows, Ubuntu, macOS) - # 2. All supported Unity Editor versions - # Package testing is comprehensive across environments because: - # - Different platforms may expose platform-specific issues - # - Editor versions may have different compatibility requirements - # - API compatibility needs validation across all supported versions + # 1. For all desktop platform (Windows, Ubuntu, macOS) defined in test_platforms.desktop (in project.metafile) + # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors defined in project.metafile) + + # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. + # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. + + # The jobs name being generated would be for example "Package Test - NGO testproject [win, 6000.0]" (name visible in Yamato) or "package_test_-_ngo_6000.0_win" (job name when querying for it) # TECHNICAL CONSTRAINTS--------------------------------------------------------------- - # - Runs in Editor context only (no player builds) - # - No scripting backend variations needed - # - Requires successful package pack job - # - Uses PVP for package validation - # - Requires specific display configuration for Ubuntu - # - Supports custom validation profiles via pvpExceptions.json - # - Extended logging enabled via UNITY_EXT_LOGGING + # This job runs in Editor context only (no player builds required) + # No scripting backend variations needed (Editor context) + # Architecture variations not applicable (Editor context) + # Requires project packaging as prerequisite (dependency job) + # Uses PVP for package validation. Specifically it looks for supported profiles which we should conform to but takes ./pvpExceptions.json file into consideration where we note our known issues related to PVP checks + +# QUALITY THOUGHTS-------------------------------------------------------------------- + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + # TODO: we should aim to replace target PVP profile from supported to gold #------------------------------------------------------------------------------------ @@ -42,12 +44,13 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: commands: - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait - # Validate packages. + # Validate PVP checks for package. - upm-pvp test --unity .Editor --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --results pvp-results - upm-pvp require {% if platform.name == "win" %}"%XRAY_PROFILE%"{% else %}"$XRAY_PROFILE"{% endif %} --results pvp-results + # Run UTR to test packages. - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --unity .Editor - - {% if platform.name == "ubuntu" %}DISPLAY=:0 {% endif %} UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --extra-editor-arg=-testCategory --extra-editor-arg=!Performance --reruncount=1 --clean-library-on-rerun + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --reruncount=1 --clean-library-on-rerun artifacts: logs: paths: diff --git a/.yamato/performance-tests.yml b/.yamato/performance-tests.yml index 66e37de536..09ba2c2e06 100644 --- a/.yamato/performance-tests.yml +++ b/.yamato/performance-tests.yml @@ -2,29 +2,29 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # Executes performance tests for NGO package without data reporting - # Tests run in Editor context using default project configuration - # Focuses on validating performance test execution rather than metrics collection - # Serves as a validation step to ensure performance tests are functioning correctly - # Note: This configuration explicitly disables performance data reporting + # This job executes performance tests for NGO package. + # TODO: Currently NGO don't have any performance tests so this job is a placeholder for the future + # This configuration is set to not report any data (at least in current state since we don't have any tests to run) # CONFIGURATION STRUCTURE-------------------------------------------------------------- + # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: - # 1. All desktop platforms (Windows, Ubuntu, macOS) - # 2. All supported Unity Editor versions - # 3. Default project only (testproject) - # Comprehensive platform/editor coverage ensures test stability - # Uses default project to maintain consistent test environment + # 1. For all desktop platforms (Windows, Ubuntu, macOS) + # 2. For all supported Unity Editor versions (For NGOv2.X it means 6000+) + # 3. For the default project (from project.metafile). In this case the default project (testptoject) is used as a context for the build. + + # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. + # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. + + # The jobs name being generated would be for example "Performance editor Tests - NGO testproject [win, 6000.0] (No Data Reporting)" (name visible in Yamato) or "performance_editor_tests_-_NGO_testproject_6000.0_no_data_reporting" (job name when querying for it) # TECHNICAL CONSTRAINTS--------------------------------------------------------------- - # - Requires Unity Editor installation - # - Tests run in Editor context only - # - 3600 second (1 hour) timeout limit - # - Focuses on Unity.NetCode.* assemblies - # - Performance category tests only - # - Disables package manager traces - # - Includes extensive log collection - # - No performance metrics are reported to monitoring systems + # Tests run in Editor context only + # No performance metrics are reported to monitoring systems + +# QUALITY THOUGHTS-------------------------------------------------------------------- + # We should discuss how to approach the topic of performance testing + # To see where this job is included (in trigger job definitions) look into _triggers.yml file #------------------------------------------------------------------------------------ @@ -38,25 +38,15 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - npm install -g upm-ci-utils@stable --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - # Installing editor - - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait + - unity-downloader-cli -u {{ editor }} -c Editor # Build Player - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata artifacts: logs: paths: - - '*.log' - - '*.xml' - - artifacts/**/* - - {{ project.name }}/Logs/** - - {{ project.name }}/Library/*.log - - {{ project.name }}/*.log - - {{ project.name }}/Builds/*.log - - build/test-results/** + - "artifacts/**/*" {% endfor -%} {% endfor -%} {% endfor -%} \ No newline at end of file diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml index 87517b1589..6cd6ea42f0 100644 --- a/.yamato/project-pack.yml +++ b/.yamato/project-pack.yml @@ -2,40 +2,46 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # Defines project packaging jobs for NGO projects - # These jobs prepare project packages that will be consumed by other pipeline jobs - # While these jobs don't require Unity Editor, they're distributed across platforms to prevent bottlenecks that would occur if all packaging was done on minimal VMs + # This job is responsible for packing a specific project. It generates package artifacts (.tgz) required for testing and publishing, ensuring all dependencies are properly bundled and validated before any test execution. + # The job itself doesn't test anything specific but rather it prepares project packages that will be consumed by other pipeline jobs. # CONFIGURATION STRUCTURE-------------------------------------------------------------- + # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: - # 1. All NGO projects (from project.metafile) - # 2. Desktop platforms (Windows, Ubuntu, macOS) - # This allows even distribution of packaging workload across different platforms - # Each configuration creates a separate packaging job for better parallelization + # 1. For every NGO project defined in projects.all (in project.metafile) + # 2. For all desktop platform (Windows, Ubuntu, macOS) defined in test_platforms.desktop (in project.metafile) -# TECHNICAL CONSTRAINTS---------------------------------------------------------------- - # - No Unity Editor instance required - # - No specific platform dependencies - # - Could use minimal VM resources but distributed for better throughput - # - Requires npm and upm-ci-utils package - # - Generates artifacts in upm-ci~/packages directory + # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. + # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. + + # The jobs name being generated would be for example "Project Pack - testproject [win]" (name visible in Yamato) or "project_pack_-_testproject_win" (job name when querying for it) + +# TECHNICAL ASPECTS-------------------------------------------------------------------- + # Job does not require Unity Editor in order to perform packing. + # upm-ci is being used (already preinstalled on images that we use) but in the future it should be replaced by upm-pvp. + # In theory, we could just use one platform for packing projects (for example ubuntu) but in order to reduce confusion we are using same platform as the job utilizing project pack as dependency. + # Job generates artifacts in upm-ci~/packages directory + +# QUALITY THOUGHTS-------------------------------------------------------------------- + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + # TODO: we should investigate adding PVP checks into project pack job and generally replacing upm-ci with upm-pvp #-------------------------------------------------------------------------------------- -{% for project in projects.all -%} # Jobs are being defined per each project as in project.metafile (testproject, minimalproject, testproject-tools-integration). -{% for platform in test_platforms.desktop -%} # Jobs are being defined per each desktop platform as in project.metafile declaration (Win, Ubuntu, Mac) -project_pack_-_{{ project.name }}_{{ platform.name }}: # This is a job name when querying from code (for example in _run-all.yml). Notice that parts of it will be filled in from for loops. For example there will be a configuration "project_pack_-_testproject_win" and when querying for this job we could for example change some values to get a given job (for example change platform value) and this will query different job configuration. - name: Project Pack - {{ project.name }} [{{ platform.name }}] # This is a job name visible in Yamato (notice that here we also use "generated" fields from for loops +{% for project in projects.all -%} +{% for platform in test_platforms.desktop -%} +project_pack_-_{{ project.name }}_{{ platform.name }}: + name: Project Pack - {{ project.name }} [{{ platform.name }}] agent: - type: {{ platform.type }} # Defines type from platform configuration (for example Unity::VM). For packing no GPU extension is needed - image: {{ platform.image }} # Bokken image specification - flavor: {{ platform.flavor }} # Memory/CPU allocation configuration + type: {{ platform.type }} + image: {{ platform.image }} + flavor: {{ platform.flavor }} commands: - - upm-ci project pack --project-path {{ project.path }} # upm-ci version of packing command (will be replaced by upm-pvp in the future) + - upm-ci project pack --project-path {{ project.path }} artifacts: packages: paths: - - "upm-ci~/packages/**/*" # Generates artifacts in upm-ci~/packages directory + - "upm-ci~/packages/**/*" {% endfor -%} {% endfor -%} \ No newline at end of file diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index 53f2c6c331..ef3afa55ab 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -2,29 +2,34 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # Validates project compliance with NGO coding standards and conventions - # While testproject validation would be sufficient, all projects are checked - # to ensure consistent code quality across the entire codebase + # This job validates project compliance with NGO coding standards and conventions # Standards validation includes: - # - Code formatting compliance - # - Project structure validation - # - Coding convention adherence - # - Solution file synchronization + # - Code formatting compliance + # - Project structure validation + # - Coding convention adherence + # - Solution file synchronization # CONFIGURATION STRUCTURE-------------------------------------------------------------- + # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: - # 1. All projects (comprehensive validation) - # 2. Default platform only (Ubuntu for efficiency) - # 3. Default editor version (trunk) - # Results are consistent across platforms/editors, so default configurations - # are used to optimize CI resource usage + # 1. For every NGO project defined in projects.all (in project.metafile) + # 2. For default platform only (Ubuntu) since standards would not vary between platforms (no need for checks on more platforms) + # 3. For default editor version (trunk) since standards would not vary between editors (no need for checks on more editors) + + # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. + # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. + + # The jobs name being generated would be for example "Standards Check - NGO testproject [win, 6000.0]" (name visible in Yamato) or "standards_win_testproject_6000.0" (job name when querying for it) # TECHNICAL CONSTRAINTS--------------------------------------------------------------- - # - Requires .NET SDK installation + # - Requires .NET SDK installed (should be preinstalled on the image) # - Needs Unity Editor for solution synchronization # - Uses custom standards validation tool (netcode.standards) - # - Platform/editor independent validation - # - Generates no test artifacts (pass/fail only) + # - Generates no test artifacts (pass/fail only). Eventual failure will be visible in the logs + +# QUALITY THOUGHTS-------------------------------------------------------------------- + # While testproject validation would be sufficient (since it validates both project and package which is our main concern), jobs for all projects are being generated to ensure that we conform to quality standards in all projects. + # To see where this job is included (in trigger job definitions) look into _triggers.yml file #------------------------------------------------------------------------------------ @@ -43,12 +48,11 @@ standards_{{ platform.name }}_{{ project.name }}_{{ editor }}: - dotnet format --version # Unity setup - - pip install unity-downloader-cli --upgrade --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - unity-downloader-cli -u {{ editor }} -c editor --wait --fast - .Editor/Unity -batchmode -nographics -logFile - -executeMethod Packages.Rider.Editor.RiderScriptEditor.SyncSolution -projectPath {{ project.path }} -quit # Standards check - - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check + - dotnet run --project=dotnet-tools/netcode.standards --project={{ project.path }} --check {% endfor -%} {% endfor -%} {% endfor -%} \ No newline at end of file diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 0bc2c6cdb8..8fea852830 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -2,27 +2,30 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # Executes project-specific tests in Unity Editor context - # Covers both PlayMode and EditMode tests from project test assemblies - # Excludes NGO package tests (handled in separate package test jobs) - # Provides validation across all supported Unity Editor versions - # Note: These tests focus on project-specific functionality, not package validation + # This job executes project-specific tests in Unity Editor context + # Those tests cover both PlayMode and EditMode tests from project test assemblies + # We need to note that NGO package tests are not being executed within this job (those are handled in separate package test jobs) # CONFIGURATION STRUCTURE-------------------------------------------------------------- + # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: - # 1. All projects with tests (filtered by has_tests flag) - # 2. All desktop platforms (Windows, Ubuntu, macOS) - # 3. All supported Unity Editor versions - # This ensures comprehensive testing across supported environments - # Each combination creates a separate test job for parallel execution + # 1. For every NGO project defined in projects.all (in project.metafile) with tests (filtered by has_tests flag) [For more info look into project.metafile configuration] + # 2. For all desktop platform (Windows, Ubuntu, macOS) defined in test_platforms.desktop (in project.metafile) + # 3. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors defined in project.metafile) + + # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. + # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. + + # The jobs name being generated would be for example "Project Test - NGO testproject [win, 6000.0]" (name visible in Yamato) or "test_testproject_win_6000.0" (job name when querying for it) # TECHNICAL CONSTRAINTS--------------------------------------------------------------- - # - Runs in Editor context only (no player builds required) - # - No scripting backend variations needed (Editor uses Mono) - # - Architecture variations not applicable (Editor context) - # - Requires project packaging as prerequisite - # - Special handling for Ubuntu display configuration - # - Excludes performance tests for consistent CI execution + # This job runs in Editor context only (no player builds required) + # No scripting backend variations needed (Editor context) + # Architecture variations not applicable (Editor context) + # Requires project packaging as prerequisite (dependency job) + +# QUALITY THOUGHTS-------------------------------------------------------------------- + # To see where this job is included (in trigger job definitions) look into _triggers.yml file #------------------------------------------------------------------------------------ @@ -37,15 +40,14 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait - - {% if platform.name == "ubuntu" %}DISPLAY=:0 && {% endif %}upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--extra-editor-arg=-testCategory --extra-editor-arg=!Performance --reruncount=1 --clean-library-on-rerun" + - unity-downloader-cli -u {{ editor }} -c Editor + - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--reruncount=1 --clean-library-on-rerun" artifacts: logs: paths: - - "upm-ci~/test-results/**/*" # Test execution logs and results + - "upm-ci~/test-results/**/*" dependencies: - - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} # Requires packaged project to be available + - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} {% endfor -%} {% endfor -%} {% endif -%} diff --git a/.yamato/project-updated-dependencies-test.yml b/.yamato/project-updated-dependencies-test.yml index b06b23623a..94b3754a7d 100644 --- a/.yamato/project-updated-dependencies-test.yml +++ b/.yamato/project-updated-dependencies-test.yml @@ -3,43 +3,50 @@ # DESCRIPTION-------------------------------------------------------------------------- # Validates package compatibility with latest dependency versions - # Tests each package referenced in the project against newest compatible dependency versions + # Tests each package referenced in the project against the newest compatible dependency versions # Helps detect potential breaking changes from dependency updates early - # Runs on default editor/platform to balance between coverage and resource usage # CONFIGURATION STRUCTURE-------------------------------------------------------------- + # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: - # 1. Default projects only (optimized subset from project.metafile) - # 2. Default platform (typically Ubuntu for resource efficiency) - # 3. Default editor version (latest stable) - # This focused configuration allows quick validation while managing resource usage + # 1. Jobs are being defined per each project as in project.metafile (testproject, minimalproject, testproject-tools-integration). + # 2. Jobs are being defined per each desktop platform as in project.metafile declaration (Win, Ubuntu, Mac) + # 3. Jobs are being defined per each supported editor (For NGOv2.X it means 6000+) + + # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. + # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. + + # The jobs name being generated would be for example "Updated Dependencies Test - NGO testproject [win, 6000.0]" (name visible in Yamato) or "updated-dependencies_testproject_NGO_win_6000.0" (job name when querying for it) # TECHNICAL CONSTRAINTS---------------------------------------------------------------- - # - Requires successful project packaging before execution - # - Tests only NGO package dependencies (com.unity.netcode.gameobjects) - # - Uses default configurations to optimize CI resource usage - # - Generates test results in upm-ci~/test-results directory - # - Dependencies must follow semantic versioning for proper version resolution + # This job requires successful project packaging before execution (job dependency) + # This job tests only NGO package dependencies (com.unity.netcode.gameobjects) + # The results are being generated in upm-ci~/test-results directory (specific of upm-ci) + # Currently upm-ci is being used but in the future it will be replaced by upm-pvp + +# QUALITY THOUGHTS--------------------------------------------------------------------- + # To see where this job is included (in trigger job definitions) look into _triggers.yml file #-------------------------------------------------------------------------------------- -{% for project in projects.all -%} # Jobs are being defined per each project as in project.metafile (testproject, minimalproject, testproject-tools-integration). -{% for platform in test_platforms.desktop -%} # Jobs are being defined per each desktop platform as in project.metafile declaration (Win, Ubuntu, Mac) -{% for editor in validation_editors.all -%} # Jobs are being defined per each supported editor (For NGOv2.X it means 6000+) -updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: +{% for project in projects.all -%} +{% for platform in test_platforms.desktop -%} +{% for editor in validation_editors.all -%} +updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: name : Updated Dependencies Test - NGO {{ project.name }} [{{ platform.name }}, {{ editor }}] agent: - type: {{ platform.type }} # Defines type from platform configuration (for example Unity::VM). For packing no GPU extension is needed - image: {{ platform.image }} # Bokken image specification - flavor: {{ platform.flavor }} # Memory/CPU allocation configuration + type: {{ platform.type }} + image: {{ platform.image }} + flavor: {{ platform.flavor }} commands: - - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type updated-dependencies-tests --package-filter com.unity.netcode.gameobjects # Execute dependency update tests with specific package filter + # Execute dependency update tests with specific package filter (we only care about NGO package) + - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type updated-dependencies-tests --package-filter com.unity.netcode.gameobjects artifacts: logs: paths: - - "upm-ci~/test-results/**/*" # Test execution logs and results + - "upm-ci~/test-results/**/*" dependencies: - - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} # Requires packaged project to be available + - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} {% endfor -%} {% endfor -%} {% endfor -%} diff --git a/.yamato/project.metafile b/.yamato/project.metafile index ff48f37fdb..7607fd7fa3 100644 --- a/.yamato/project.metafile +++ b/.yamato/project.metafile @@ -1,11 +1,21 @@ -# DESCRIPTION-------------------------------------------------------------------------- -# Central configuration file defining test environments, platforms, and projects -# Used to generate CI pipeline configurations through templating -# Provides consistent environment definitions across all CI jobs -# Optimizes resource usage through appropriate agent selection +# DESCRIPTION----------------------------------------------------------------------------------------------------------------- +# This is the central configuration file defining test environments, platforms, and projects. +# This is used to generate CI pipeline configurations through templating +# Provides consistent environment definitions across all CI jobs. It's easier to modify values in central place (for example image reference) than having to modify every job -# AGENT CONFIGURATIONS--------------------------------------------------------------- -# The small agent was created to handle jobs that don't involve running Unity (for example pack job). +# Usual fields being defined are + # name --> Identifier used to distinguish different configurations in the CI pipeline + # type --> Specifies the Bokken agent type (e.g., Unity::VM, Unity::VM::osx, Unity::mobile::shield) + # image --> Defines the package-ci Bokken image to use for the environment (e.g., package-ci/ubuntu-22.04:v4). This is basically a device configuration + # flavor --> Determines the VM size/resources (e.g., b1.small, b1.large, m1.mac) + # standalone --> Specifies the build target platform (e.g., StandaloneLinux64, Android, IOS) + # model --> Defines specific hardware model requirements (e.g., rtx2080, SE for iPhone SE) + # base --> Indicates the base operating system for build operations (e.g., win, mac) + # architecture --> Specifies the target CPU architecture (e.g., armv7, arm64) + + +# SMALL AGENT CONFIGURATION--------------------------------------------------------------------------------------------------- +# The small agent was created to handle jobs that don't involve running Unity and are in general super light when it comes to resource usage (for example pack job). # It uses ubuntu since Linux VMs are faster and cheaper to provision than Mac or Windows Virtual Machines (VMs). # The b1.small flavour is nearly always sufficient for jobs that don’t involve running Unity Editor. small_agent_platform: @@ -14,8 +24,10 @@ small_agent_platform: image: package-ci/ubuntu-22.04:v4 flavor: b1.small -# PLATFORM CONFIGURATIONS----------------------------------------------------------- -# Platforms that project will/can be tested on. + +# PLATFORM CONFIGURATIONS----------------------------------------------------------------------------------------------------- +# test_platforms specifies platforms that project will/can be tested on. +# For desktops we include Windows, Ubuntu and Mac, for mobiles we include Android and iOS, for consoles we include ps4, ps5, switch, xbox360 and xboxOne test_platforms: # Default platform is used for all basic jobs. Ubuntu was chosen since it's fastest and most available default: @@ -43,6 +55,7 @@ test_platforms: flavor: m1.mac standalone: StandaloneOSX # For mobile devices there is a split between the build and run phase so there is a need of splitting specification for both + # TODO: For android we could consider adding ARM64 configuration mobile_build: - name: android type: Unity::VM @@ -51,7 +64,7 @@ test_platforms: standalone: Android base: win architecture: armv7 - # iOS modern builds are ARM64-only, thus no testing with armv7 (as in android case) + # iOS modern builds are ARM64-only, thus no testing with armv7 - name: ios-arm64 type: Unity::VM::osx image: package-ci/macos-13:v4 @@ -129,9 +142,8 @@ test_platforms: flavor: b1.large standalone: GameCoreScarlett -# EDITOR CONFIGURATIONS------------------------------------------------------------ -# Editors to be used for testing. -# Since NGOv2 official support started from U6 it means that only those editors should be used for testing +# EDITOR CONFIGURATIONS------------------------------------------------------------------------------- +# Editors to be used for testing. NGOv2.X official support started from 6000.0 editor validation_editors: default: - trunk @@ -141,15 +153,16 @@ validation_editors: - trunk -# Scripting backends used by Standalone RunTimeTests +# Scripting backends used by Standalone RunTimeTests--------------------------------------------------- scripting_backends: - mono - il2cpp -# Projects within the repository that will be tested. Name will be used -# for job ids, so it should not contain spaces/non-supported characters -# The default project will also be used for validation, code coverage etc. +# PROJECTS CONFIGURATIONS------------------------------------------------------------------------------- +# Projects within the repository that will be tested. +# The default project should be used for general validation, code coverage and other tests where choice of project should not matter (we are interested in package) +# has_tests --> describes if projects contains any tests to run. projects: default: - name: testproject diff --git a/.yamato/webgl-build.yml b/.yamato/webgl-build.yml index 53c61d6ac2..cc0a577f40 100644 --- a/.yamato/webgl-build.yml +++ b/.yamato/webgl-build.yml @@ -1,41 +1,45 @@ -{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- # DESCRIPTION-------------------------------------------------------------------------- - # This job definition creates job configurations to test building a player on WebGl standalone platform. + # This job is responsible for validating a successful building of a player on WebGl standalone platform. # This job will not execute any tests, we only test building process. - # This setup performs build-only validation (without running tests) because WebGL runs in browser and for tests to be executed we would need to consider having a web server, browser automation and overall complex test setup which currently we don't have. - - # The setup below defines multiple jobs with different configurations (taken from project.metafile in the 1st line). - # This is a Yamato specific as it will "compile" this script and generate given configurations which we can easily modify from within project.metafile. - # From there (project.metafile) we could easily add for example a new validation editor or remove unsupported one. + # This is because WebGL runs in browser and for tests to be executed we would need to consider having a web server, browser automation and overall complex test setup which currently we don't have. # CONFIGURATION STRUCTURE-------------------------------------------------------------- + # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: - # 1. Projects (from project.metafile) - # 2. Desktop platforms (Windows, Ubuntu, macOS) - # 3. Supported Unity Editor versions - # This structure allows easy modification of supported configurations via project.metafile + # 1. For the default project (from project.metafile). In this case the default project (testptoject) is used as a context for the build. + # 2. For all desktop platforms (Windows, Ubuntu, macOS) + # 3. For all supported Unity Editor versions (For NGOv2.X it means 6000+) + + # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. + # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. + + # The jobs name being generated would be for example "WebGl Build - testproject [win, 6000.0, il2cpp]" (name visible in Yamato) or "webgl_build_testproject_win_6000.0" (job name when querying for it) # TECHNICAL CONSTRAINTS---------------------------------------------------------------- # WebGL requires IL2CPP scripting backend (Mono not supported) - # x64 architectures preferred for build phase (ARM64 not used to optimize resource usage. For build phase x64 architectures are preferred) - # Build validation only (no runtime testing) - - + # We are not using ARM64 architectures since we only perform a build action. x64 architectures are preferred for build phase (in order to optimize available resource usage) + # We only perform build validation (no runtime testing) + +# QUALITY THOUGHTS-------------------------------------------------------------------- + # In the future we could try to implement an infrastructure to run test in webgl context but this could be quite complicated and would need to be evaluated if it's worth it + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + #-------------------------------------------------------------------------------------- -{% for project in projects.default -%} # Default project (testptoject) in this case is used as a context for the build. -{% for platform in test_platforms.desktop -%} # Jobs are being defined per each desktop platform as in project.metafile declaration (Win, Ubuntu, Mac) -{% for editor in validation_editors.all -%} # Jobs are being defined per each supported editor (For NGOv2.X it means 6000+) -webgl_build_{{ project.name }}_{{ platform }}_{{ editor }}: # This is a job name when querying from code (for example in _run-all.yml). Notice that parts of it will be filled in from for loops. For example there will be a configuration "webgl_build_testproject_win_6000.0" and when querying for this job we could for example change some values to get a given job (for example change editor value) and this will query different job configuration - name: WebGl Build - {{ project.name }} [{{ platform.name }}, {{ editor }}, il2cpp] # This is a job name visible in Yamato (notice that here we also use "generated" fields from for loops +{% for project in projects.default -%} +{% for platform in test_platforms.desktop -%} +{% for editor in validation_editors.all -%} +webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: + name: WebGl Build - {{ project.name }} [{{ platform.name }}, {{ editor }}, il2cpp] agent: - type: {{ platform.type }} # Defines type from platform configuration (for example Unity::VM). For webgl builds no GPU extension is needed - image: {{ platform.image }} # Bokken image specification - flavor: {{ platform.flavor }} # Memory/CPU allocation configuration + type: {{ platform.type }} + image: {{ platform.image }} + flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor -c webgl -c il2cpp --wait # unity-downloader-cli should be already preinstalled on the image. This steps downloads proper editor to build from together with required components (webgl + il2cpp). The --wait flag will ensure that we wait until editor is downloaded + - unity-downloader-cli -u {{ editor }} -c Editor -c webgl -c il2cpp # The following step builds the player with defined options such as: # Suite parameter if defined since it's a mandatory field to define which test suite should be used, but it doesn't matter in this case since we won't run any tests (--suite) @@ -50,8 +54,8 @@ webgl_build_{{ project.name }}_{{ platform }}_{{ editor }}: artifacts: logs: paths: - - "artifacts/**/*" # Build process logs (path defined via --artifacts_path argument) - - "build/players/**/*" # WebGL build output files (path defined via --player-save-path argument) + - "artifacts/**/*" + - "build/players/**/*" {% endfor -%} {% endfor -%} {% endfor -%} From 9d1fcd820f1c5881f9fb0b85a5f48f3571192887 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Tue, 4 Mar 2025 17:30:16 +0100 Subject: [PATCH 15/39] Corrected package pack command --- .yamato/project-pack.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml index 6cd6ea42f0..112fde8474 100644 --- a/.yamato/project-pack.yml +++ b/.yamato/project-pack.yml @@ -38,6 +38,7 @@ project_pack_-_{{ project.name }}_{{ platform.name }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: + - npm install -g upm-ci-utils@stable --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - upm-ci project pack --project-path {{ project.path }} artifacts: packages: From 9869eeddec8022856f9d690fdb475fc518af5144 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Wed, 5 Mar 2025 10:59:07 +0100 Subject: [PATCH 16/39] Corrected upm-ci installs --- .yamato/project-pack.yml | 2 +- .yamato/project-standards.yml | 2 +- .yamato/project-tests.yml | 5 +++-- .yamato/project-updated-dependencies-test.yml | 1 + 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml index 112fde8474..78d21a7766 100644 --- a/.yamato/project-pack.yml +++ b/.yamato/project-pack.yml @@ -38,7 +38,7 @@ project_pack_-_{{ project.name }}_{{ platform.name }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - npm install -g upm-ci-utils@stable --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm + - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - upm-ci project pack --project-path {{ project.path }} artifacts: packages: diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index ef3afa55ab..640d631af5 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -52,7 +52,7 @@ standards_{{ platform.name }}_{{ project.name }}_{{ editor }}: - .Editor/Unity -batchmode -nographics -logFile - -executeMethod Packages.Rider.Editor.RiderScriptEditor.SyncSolution -projectPath {{ project.path }} -quit # Standards check - - dotnet run --project=dotnet-tools/netcode.standards --project={{ project.path }} --check + - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check {% endfor -%} {% endfor -%} {% endfor -%} \ No newline at end of file diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 8fea852830..856095a6b9 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -40,8 +40,9 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor - - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--reruncount=1 --clean-library-on-rerun" + - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it + - unity-downloader-cli -u {{ editor }} -c Editor # Installing basic editor for tests execution + - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--reruncount=1 --clean-library-on-rerun" # project tests execution via upm-ci artifacts: logs: paths: diff --git a/.yamato/project-updated-dependencies-test.yml b/.yamato/project-updated-dependencies-test.yml index 94b3754a7d..74f0d18c27 100644 --- a/.yamato/project-updated-dependencies-test.yml +++ b/.yamato/project-updated-dependencies-test.yml @@ -39,6 +39,7 @@ updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: + - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it # Execute dependency update tests with specific package filter (we only care about NGO package) - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type updated-dependencies-tests --package-filter com.unity.netcode.gameobjects artifacts: From e8ba41bb2bbb7ccbf6535cad85cea160d9de3824 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Wed, 5 Mar 2025 13:23:21 +0100 Subject: [PATCH 17/39] Second comment pass --- .yamato/_run-all.yml | 12 +++++-- .yamato/_triggers.yml | 19 ++++++---- .yamato/code-coverage.yml | 13 +++---- .yamato/console-standalone-test.yml | 27 ++++++++------ .yamato/desktop-standalone-tests.yml | 26 +++++++++----- .yamato/mobile-standalone-test.yml | 35 +++++++++--------- .yamato/package-pack.yml | 17 ++++++--- .yamato/package-tests.yml | 12 +++---- .yamato/performance-tests.yml | 26 +++++++------- .yamato/project-pack.yml | 16 ++++----- .yamato/project-standards.yml | 36 +++++++++---------- .yamato/project-tests.yml | 19 +++++----- .yamato/project-updated-dependencies-test.yml | 22 ++++++------ .yamato/project.metafile | 20 +++++++---- .yamato/webgl-build.yml | 21 +++++------ 15 files changed, 179 insertions(+), 142 deletions(-) diff --git a/.yamato/_run-all.yml b/.yamato/_run-all.yml index f031dc7ff7..ab056a94c5 100644 --- a/.yamato/_run-all.yml +++ b/.yamato/_run-all.yml @@ -2,8 +2,16 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # Those jobs group together related test executions - # Enables efficient test execution for different validation scenarios + # Those jobs group together related test as dependencies to allow to easily manage running a given set of tests. + # This enables efficient test execution for different validation scenarios + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # Jobs are generated using nested loops through: + + # It's important to ensure that all generated in such way dependencies exist (this can be verified in Yamato) + # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. + # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. #----------------------------------------------------------------------------------- diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index 89d8149ad1..436d51787b 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -2,30 +2,37 @@ --- # DESCRIPTION-------------------------------------------------------------------------- -# Defines three main CI trigger patterns: - # 1. Pull Request Validation: Basic validation for PR changes - # 2. Nightly Development: Comprehensive testing on trunk - # 3. Weekly Full Validation: Complete testing across all editor versions -# Each pattern represents different balance between validation depth, execution time and CI resource usage + # This configuration defines three main CI trigger patterns: + # 1. Pull Request Validation: Validation performed on PR basis + # 2. Nightly Development: Test set run nightly (validates most important test cases) + # 3. Weekly Full Validation: Test set run weekly (validates all test cases to prevent any suprises) + # Each pattern represents different balance between validation depth, execution time and CI resource usage # TRIGGER PATTERNS------------------------------------------------------------------- # Pull Request: + # This test validaes Standards, Package tests, Project tests and Desktop standalone tests to ensure that main platforms are covered # Triggers on PRs to develop, develop-2.0.0, and release branches # Focuses on critical validation paths that we should validate before merging PRs # Cancels previous runs on new commits # Excludes draft PRs # Nightly: + # This test validaes same subset as pull_request_trigger with addition of mobile/console tests and webgl builds # Runs daily on develop-2.0.0 (local configuration) # Includes all test types but only on trunk. TODO: Add validation for minimum supported editor in nightly # Adds platform-specific and APV validation # Weekly: - # Most comprehensive validation + # This test validaes same subset as develop_nightly but runs per all supported editors as well as executes code coverage test and runs project standards per project # Runs across all supported editor versions # Includes code coverage analysis # Validates all projects and standards +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs configurations are generated by ensuring that all dependencies are successful. + # The dependencies are taken from _run-all.yml file where we can gather multiple tests into proper sets + + #----------------------------------------------------------------------------------- # Run all relevant tasks when a pull request targeting the develop or release branch is created or updated. diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 3fd029adae..b2412f6c8e 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -2,7 +2,7 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # This job executes package tests with code coverage analysis enabled for the NGO (Netcode for GameObjects) package + # This job is responsible for executing package tests with code coverage analysis enabled. # Coverage analysis provides insights into: # Test coverage metrics for NGO assemblies # Line and branch coverage statistics @@ -10,7 +10,7 @@ # Additional metrics for coverage analysis # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: # 1. For default platform only (Ubuntu) since coverage would not vary between platforms (no need for checks on more platforms) # 2. For default editor version (trunk) since coverage would not vary between editors (no need for checks on more editors) @@ -20,13 +20,14 @@ # The jobs name being generated would be for example "Code Coverage - NGO [testproject, 6000.0]" (name visible in Yamato) or "code_coverage_testproject_6000.0" (job name when querying for it) -#TECHNICAL CONSTRAINTS--------------------------------------------------------------- +#TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # In theory this job also runs package tests, but we don't want to use it as default since is heavier (because of added coverage analysis) and coverage is not changing that often # Requires Unity Editor installation - # Uses UnifiedTestRunner (UTR) for test execution - # Depends on successful package pack job completion # Burst compilation is disabled to ensure accurate coverage measurement + # In order to properly use -coverage-results-path parameter we need to start it with $PWD (which means the absolute path). Otherwise coverage results will not be visible -# QUALITY THOUGHTS-------------------------------------------------------------------- +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- + # TODO: somewhere in 2025 we will be able to upload resuls to CodeCov from public repos # To see where this job is included (in trigger job definitions) look into _triggers.yml file diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index b6027e1efc..fe3b3c3ef6 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -2,23 +2,30 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # This file contains Console platform test validation for NGO package consisting of two phases: - # 1. Build Phase: Creates standalone players for console platforms - # 2. Run Phase: Executes runtime tests on actual console devices - # Those phases need to be separated (in contrast to for example desktop tests) + # This job is responsible for Console platform test validation. + # Those tests cover both PlayMode and EditMode tests from package test assemblies. # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: - # 1. For all console platform (Switch, ps4, ps5, xbox360, xboxOne) defined in test_platforms.console_ (in project.metafile) - # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors defined in project.metafile) - # 3. For default project which serves as context + # 1. For all console platform (Switch, ps4, ps5, xbox360, xboxOne) + # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) + ` # 3. For the default project. # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. # The jobs name being generated would be for example "Build testproject - [win, 6000.0, il2cpp]" (name visible in Yamato) or "console_standalone_build_testproject_win_6000.0" (job name when querying for it) +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # For console devices a split is required into two phases: + # 1. Build Phase: Creates standalone players for console platforms + # 2. Run Phase: Executes runtime tests on actual console devices + # The Run phase uses build job as dependency + + # Note: More of a Unity specific but test assemblies need to be included in the build phase command + # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices + # PLATFORM SPECIFICS----------------------------------------------------------------- # Common Requirements: # All consoles require IL2CPP scripting backend @@ -28,11 +35,9 @@ # Switch: ARM64 architecture only # Other Consoles: x64 architecture # Each console requires specific SDK paths and tools - - # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices - # Note: More of a Unity specific but test assemblies need to be included in the build phase command # QUALITY THOUGHTS-------------------------------------------------------------------- + # TODO: consider adding all projects that have tests # To see where this job is included (in trigger job definitions) look into _triggers.yml file diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index a47a136e56..430aeefc37 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -2,25 +2,33 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # This file contains Desktop platform test validation for NGO package consisting of two phases: - # 1. Build Phase: Creates standalone players for desktop platforms - # 2. Run Phase: Executes runtime tests on actual desktop devices - # For desktop devices the phase separation is not necessary, but we did it for consistency with other standalone tests configurations + # This job is responsible for Desktop platform test validation. + # Those tests cover both PlayMode and EditMode tests from package test assemblies. # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: - # 1. For all desktop platform (Windows, macOS, Ubuntu) defined in test_platforms.desktop_ (in project.metafile) - # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors defined in project.metafile) - # 3. For default project which serves as context - # 4. For all scripting backends + # 1. For all desktop platform (Windows, macOS, Ubuntu) + # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) + # 3. For the default project. + # 4. For all scripting backends (mono, il2cpp) # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. # The jobs name being generated would be for example "Standalone Build - NGO testproject - [win, mono, 6000.0]" (name visible in Yamato) or "desktop_standalone_build_testproject_win_mono_6000.0" (job name when querying for it) +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # For desktop devices a split is into two phases is not required but we use it for consistency with setup of others standalone platforms: + # 1. Build Phase: Creates standalone players for desktop platforms + # 2. Run Phase: Executes runtime tests on actual desktop devices + # The Run phase uses build job as dependency + + # Note: More of a Unity specific but test assemblies need to be included in the build phase command + # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices + # QUALITY THOUGHTS-------------------------------------------------------------------- + # TODO: consider adding all projects that have tests # To see where this job is included (in trigger job definitions) look into _triggers.yml file #----------------------------------------------------------------------------------- diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index 09fde91637..20ca1e139d 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -2,24 +2,30 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # This file contains Mobile platform test validation for NGO package consisting of two phases: - # 1. Build Phase: Creates standalone players for mobile platforms - # 2. Run Phase: Executes runtime tests on actual mobile devices - # Those phases need to be separated (in contrast to for example desktop tests) + # This job is responsible for Mobile platform test validation. + # Those tests cover both PlayMode and EditMode tests from package test assemblies. # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations - # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: - # 1. For all mobile platform (Android, iOS) defined in test_platforms.mobile_ (in project.metafile) - # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors defined in project.metafile) - # 3. For default project which serves as context + # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # Jobs are generated using nested loops through: + # 1. For all mobile platform (Android, iOS) + # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) + # 3. For the default project. # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. # The jobs name being generated would be for example "Build testproject - [win, 6000.0, il2cpp]" (name visible in Yamato) or "build_testproject_win_6000.0" (job name when querying for it) -# PLATFORM SPECIFICS----------------------------------------------------------------- +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # For mobile devices a split is required into two phases: + # 1. Build Phase: Creates standalone players for mobile platforms + # 2. Run Phase: Executes runtime tests on actual mobile devices + # The Run phase uses build job as dependency + # Note: More of a Unity specific but test assemblies need to be included in the build phase command + # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices + +# PLATFORM SPECIFICS-------------------------------------------------------------------- # iOS Requirements: # Must use IL2CPP scripting backend # Only supports ARM64 architecture @@ -29,11 +35,9 @@ # Uses IL2CPP scripting backend (recommended over Mono) # Supports both ARM64 and ARMv7 architectures # Can be build on any desktop platform - - # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices - # Note: More of a Unity specific but test assemblies need to be included in the build phase command -# QUALITY THOUGHTS-------------------------------------------------------------------- +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- + # TODO: consider adding all projects that have tests # To see where this job is included (in trigger job definitions) look into _triggers.yml file @@ -49,7 +53,6 @@ mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: flavor: {{ platform.flavor }} commands: - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} - - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: @@ -77,7 +80,7 @@ mobile_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} model: {{ platform.model }} {% endif %} commands: - # Installing editor. We still need the editor to run tests on standalone build + # Installing editor. We still need the editor to run tests on standalone build and for that the Editor is required - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} {% if platform.standalone == "Android" %} diff --git a/.yamato/package-pack.yml b/.yamato/package-pack.yml index db47ddcca6..fce9ba3987 100644 --- a/.yamato/package-pack.yml +++ b/.yamato/package-pack.yml @@ -5,20 +5,27 @@ # This job is responsible for packing a specific package. It generates package artifacts (.tgz) required for testing and publishing, ensuring all dependencies are properly bundled and validated before any test execution. # The job itself doesn't test anything specific but rather it prepares project packages that will be consumed by other pipeline jobs. # The job performs additional validation by using Package Verification Pipeline (PVP). It includes x-ray validation for quick package structure verification - # Because of that we can detect errors at the early stage of testing so not to waste CI resources + # Because of that validation we can detect errors at the early stage of testing so not to waste CI resources # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: - # 1. For all desktop platform (Windows, Ubuntu, macOS) defined in test_platforms.desktop (in project.metafile) + # 1. For all desktop platforms (Windows, Ubuntu, macOS) -# TECHNICAL CONSTRAINTS--------------------------------------------------------------- + # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. + # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. + + # The jobs name being generated would be for example "Package Pack (and x-ray) - NGO [win]" (name visible in Yamato) or "package_pack_-_ngo_win" (job name when querying for it) + +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # Job does not require Unity Editor in order to perform packing. # Job uses PVP x-ray for lightweight validation # Job generates both packages artifacts and pvp-results file. + # In theory, we could just use one platform for packing projects (for example ubuntu) but in order to reduce confusion we are using same platform as the job utilizing project pack as dependency. -# QUALITY THOUGHTS-------------------------------------------------------------------- +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- # To see where this job is included (in trigger job definitions) look into _triggers.yml file + # TODO: we should aim to replace target PVP profile from supported to gold #------------------------------------------------------------------------------------ diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index 49b0044727..c9185b06ae 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -2,29 +2,29 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # This job executes package-specific tests in Unity Editor context + # This job is responsible for execution of package-specific tests in Unity Editor context # Those tests cover both PlayMode and EditMode tests from package test assemblies # Additionally it combines Package Verification Pipeline (PVP) validation. This ensures that package is compatible with Unity standards # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: - # 1. For all desktop platform (Windows, Ubuntu, macOS) defined in test_platforms.desktop (in project.metafile) - # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors defined in project.metafile) + # 1. For all desktop platforms (Windows, Ubuntu, macOS) + # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. # The jobs name being generated would be for example "Package Test - NGO testproject [win, 6000.0]" (name visible in Yamato) or "package_test_-_ngo_6000.0_win" (job name when querying for it) -# TECHNICAL CONSTRAINTS--------------------------------------------------------------- +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # This job runs in Editor context only (no player builds required) # No scripting backend variations needed (Editor context) # Architecture variations not applicable (Editor context) # Requires project packaging as prerequisite (dependency job) # Uses PVP for package validation. Specifically it looks for supported profiles which we should conform to but takes ./pvpExceptions.json file into consideration where we note our known issues related to PVP checks -# QUALITY THOUGHTS-------------------------------------------------------------------- +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- # To see where this job is included (in trigger job definitions) look into _triggers.yml file # TODO: we should aim to replace target PVP profile from supported to gold diff --git a/.yamato/performance-tests.yml b/.yamato/performance-tests.yml index 09ba2c2e06..fd0df85a34 100644 --- a/.yamato/performance-tests.yml +++ b/.yamato/performance-tests.yml @@ -2,28 +2,28 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # This job executes performance tests for NGO package. - # TODO: Currently NGO don't have any performance tests so this job is a placeholder for the future - # This configuration is set to not report any data (at least in current state since we don't have any tests to run) + # This job is responsible for executing performance tests for NGO package. + # Its configuration is set to not report any data but just to give results (at least in current state since we don't have any tests to run). + # Currently because of lack of performance tests this job will always return "no tests have been selected" and because oif that it's not included in any trigger jobs. # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations - # Jobs are generated using nested loops through: + # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # Jobs configurations are generated using nested loops through: # 1. For all desktop platforms (Windows, Ubuntu, macOS) # 2. For all supported Unity Editor versions (For NGOv2.X it means 6000+) - # 3. For the default project (from project.metafile). In this case the default project (testptoject) is used as a context for the build. + # 3. For the default project (project is used only as a context for the build). TODO-comment: if performance tests would be included in projects then we should make an approperiate split. # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. # The jobs name being generated would be for example "Performance editor Tests - NGO testproject [win, 6000.0] (No Data Reporting)" (name visible in Yamato) or "performance_editor_tests_-_NGO_testproject_6000.0_no_data_reporting" (job name when querying for it) -# TECHNICAL CONSTRAINTS--------------------------------------------------------------- - # Tests run in Editor context only +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # Tests are run in Editor context only # No performance metrics are reported to monitoring systems -# QUALITY THOUGHTS-------------------------------------------------------------------- - # We should discuss how to approach the topic of performance testing +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- + # TODO: Currently NGO don't have any performance tests so this job is a placeholder for the future. We should discuss how to approach the topic of performance testing # To see where this job is included (in trigger job definitions) look into _triggers.yml file #------------------------------------------------------------------------------------ @@ -38,11 +38,9 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - # Installing editor - - unity-downloader-cli -u {{ editor }} -c Editor - - # Build Player + - unity-downloader-cli -u {{ editor }} -c Editor # Installing basic editor - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata + # TODO: when performance tests will be present we need to add actuall execution of this test artifacts: logs: paths: diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml index 78d21a7766..f2f3d2f841 100644 --- a/.yamato/project-pack.yml +++ b/.yamato/project-pack.yml @@ -6,25 +6,23 @@ # The job itself doesn't test anything specific but rather it prepares project packages that will be consumed by other pipeline jobs. # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations - # Jobs are generated using nested loops through: - # 1. For every NGO project defined in projects.all (in project.metafile) - # 2. For all desktop platform (Windows, Ubuntu, macOS) defined in test_platforms.desktop (in project.metafile) + # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # Jobs configurations are generated using nested loops through: + # 1. For all projects (testproject, minimalproject, testproject-tools-integration). + # 2. For all desktop platforms (Win, Ubuntu, Mac) # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. # The jobs name being generated would be for example "Project Pack - testproject [win]" (name visible in Yamato) or "project_pack_-_testproject_win" (job name when querying for it) -# TECHNICAL ASPECTS-------------------------------------------------------------------- +# TECHNICAL CONSIDERATIONS-------------------------------------------------------------------- # Job does not require Unity Editor in order to perform packing. - # upm-ci is being used (already preinstalled on images that we use) but in the future it should be replaced by upm-pvp. # In theory, we could just use one platform for packing projects (for example ubuntu) but in order to reduce confusion we are using same platform as the job utilizing project pack as dependency. - # Job generates artifacts in upm-ci~/packages directory -# QUALITY THOUGHTS-------------------------------------------------------------------- +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- # To see where this job is included (in trigger job definitions) look into _triggers.yml file - # TODO: we should investigate adding PVP checks into project pack job and generally replacing upm-ci with upm-pvp + # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp. Additionally this would allow us to run PVP checks on projects #-------------------------------------------------------------------------------------- diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index 640d631af5..15798cc314 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -2,17 +2,17 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # This job validates project compliance with NGO coding standards and conventions + # This job is responsible for validating project compliance with NGO coding standards and conventions # Standards validation includes: - # - Code formatting compliance - # - Project structure validation - # - Coding convention adherence - # - Solution file synchronization + # Code formatting compliance + # Project structure validation + # Coding convention adherence + # Solution file synchronization # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations - # Jobs are generated using nested loops through: - # 1. For every NGO project defined in projects.all (in project.metafile) + # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # Jobs configurations are generated using nested loops through: + # 1. For all NGO projects (testproject, testproject-tools-interation, minimalproject) # 2. For default platform only (Ubuntu) since standards would not vary between platforms (no need for checks on more platforms) # 3. For default editor version (trunk) since standards would not vary between editors (no need for checks on more editors) @@ -22,13 +22,14 @@ # The jobs name being generated would be for example "Standards Check - NGO testproject [win, 6000.0]" (name visible in Yamato) or "standards_win_testproject_6000.0" (job name when querying for it) # TECHNICAL CONSTRAINTS--------------------------------------------------------------- - # - Requires .NET SDK installed (should be preinstalled on the image) - # - Needs Unity Editor for solution synchronization - # - Uses custom standards validation tool (netcode.standards) - # - Generates no test artifacts (pass/fail only). Eventual failure will be visible in the logs + # Requires .NET SDK installed (should be preinstalled on the image so we just check for version) + # Needs Unity Editor for solution synchronization + # Uses custom standards validation tool (netcode.standards) + # Generates no test artifacts (pass/fail only). Eventual failure will be visible in the logs # QUALITY THOUGHTS-------------------------------------------------------------------- - # While testproject validation would be sufficient (since it validates both project and package which is our main concern), jobs for all projects are being generated to ensure that we conform to quality standards in all projects. + # While testproject validation would be sufficient, since it validates both project and package (where package is our main concern) jobs for all projects are being generated to ensure that we conform to quality standards in all projects. + # TODO: consider modifying the approach and adding checks for minimal supported editor. In case of NGOv1.X it has proven to yield different results (But since NGOv2.X support starts from 6000.0 editor it's not that time pressuring) # To see where this job is included (in trigger job definitions) look into _triggers.yml file #------------------------------------------------------------------------------------ @@ -47,12 +48,9 @@ standards_{{ platform.name }}_{{ project.name }}_{{ editor }}: - dotnet --version - dotnet format --version - # Unity setup - - unity-downloader-cli -u {{ editor }} -c editor --wait --fast - - .Editor/Unity -batchmode -nographics -logFile - -executeMethod Packages.Rider.Editor.RiderScriptEditor.SyncSolution -projectPath {{ project.path }} -quit - - # Standards check - - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check + - unity-downloader-cli -u {{ editor }} -c editor # Downloads basic editor + - .Editor/Unity -batchmode -nographics -logFile - -executeMethod Packages.Rider.Editor.RiderScriptEditor.SyncSolution -projectPath {{ project.path }} -quit # This command is used to invoke Unity in a "headless" mode. It's used to sync the project + - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check # Runs standards check {% endfor -%} {% endfor -%} {% endfor -%} \ No newline at end of file diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 856095a6b9..5fc3ce6a6e 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -4,27 +4,28 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job executes project-specific tests in Unity Editor context # Those tests cover both PlayMode and EditMode tests from project test assemblies - # We need to note that NGO package tests are not being executed within this job (those are handled in separate package test jobs) + # NGO package tests are NOT being executed within this job (those are handled in separate package test jobs) # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations - # Jobs are generated using nested loops through: - # 1. For every NGO project defined in projects.all (in project.metafile) with tests (filtered by has_tests flag) [For more info look into project.metafile configuration] - # 2. For all desktop platform (Windows, Ubuntu, macOS) defined in test_platforms.desktop (in project.metafile) - # 3. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors defined in project.metafile) + # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # Jobs configurations are generated using nested loops through: + # 1. For all projects WITH TESTS (filtered by has_tests flag) (testproject, testproject-tools-interation) [For more info look into project.metafile configuration] + # 2. For all desktop platforms (Windows, Ubuntu, macOS) + # 3. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. # The jobs name being generated would be for example "Project Test - NGO testproject [win, 6000.0]" (name visible in Yamato) or "test_testproject_win_6000.0" (job name when querying for it) -# TECHNICAL CONSTRAINTS--------------------------------------------------------------- - # This job runs in Editor context only (no player builds required) +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # This job runs in Editor context only (no player builds is required) # No scripting backend variations needed (Editor context) # Architecture variations not applicable (Editor context) # Requires project packaging as prerequisite (dependency job) -# QUALITY THOUGHTS-------------------------------------------------------------------- +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- + # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp # To see where this job is included (in trigger job definitions) look into _triggers.yml file #------------------------------------------------------------------------------------ diff --git a/.yamato/project-updated-dependencies-test.yml b/.yamato/project-updated-dependencies-test.yml index 74f0d18c27..80db9a15ba 100644 --- a/.yamato/project-updated-dependencies-test.yml +++ b/.yamato/project-updated-dependencies-test.yml @@ -2,29 +2,28 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # Validates package compatibility with latest dependency versions - # Tests each package referenced in the project against the newest compatible dependency versions - # Helps detect potential breaking changes from dependency updates early + # This job is responsible fo validating package compatibility with latest dependency versions. + # This helps detect potential breaking changes from dependency updates early # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations - # Jobs are generated using nested loops through: - # 1. Jobs are being defined per each project as in project.metafile (testproject, minimalproject, testproject-tools-integration). - # 2. Jobs are being defined per each desktop platform as in project.metafile declaration (Win, Ubuntu, Mac) - # 3. Jobs are being defined per each supported editor (For NGOv2.X it means 6000+) + # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # Jobs configurations are generated using nested loops through: + # 1. For all projects (testproject, minimalproject, testproject-tools-integration). + # 2. For all desktop platforms (Win, Ubuntu, Mac) + # 3. For all supported editors (For NGOv2.X it means 6000+) # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. # The jobs name being generated would be for example "Updated Dependencies Test - NGO testproject [win, 6000.0]" (name visible in Yamato) or "updated-dependencies_testproject_NGO_win_6000.0" (job name when querying for it) -# TECHNICAL CONSTRAINTS---------------------------------------------------------------- +# TECHNICAL CONSIDERATIONS---------------------------------------------------------------- # This job requires successful project packaging before execution (job dependency) # This job tests only NGO package dependencies (com.unity.netcode.gameobjects) # The results are being generated in upm-ci~/test-results directory (specific of upm-ci) - # Currently upm-ci is being used but in the future it will be replaced by upm-pvp -# QUALITY THOUGHTS--------------------------------------------------------------------- +# QUALITY CONSIDERATIONS--------------------------------------------------------------------- + # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp # To see where this job is included (in trigger job definitions) look into _triggers.yml file #-------------------------------------------------------------------------------------- @@ -40,7 +39,6 @@ updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: flavor: {{ platform.flavor }} commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - # Execute dependency update tests with specific package filter (we only care about NGO package) - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type updated-dependencies-tests --package-filter com.unity.netcode.gameobjects artifacts: logs: diff --git a/.yamato/project.metafile b/.yamato/project.metafile index 7607fd7fa3..65cab054eb 100644 --- a/.yamato/project.metafile +++ b/.yamato/project.metafile @@ -1,8 +1,9 @@ # DESCRIPTION----------------------------------------------------------------------------------------------------------------- -# This is the central configuration file defining test environments, platforms, and projects. -# This is used to generate CI pipeline configurations through templating -# Provides consistent environment definitions across all CI jobs. It's easier to modify values in central place (for example image reference) than having to modify every job +# This file is the central configuration file defining test environments, platforms, projects, editors and other parameters used. +# Data from this file is used to generate CI pipeline configurations through templating in other .yamato/ files. +# This provides consistent environment definitions across all CI jobs. It's easier to modify values in central place (for example image reference) than having to modify every job +# VALUE MEANING---------------------------------------------------------------------------------------------------------------------------- # Usual fields being defined are # name --> Identifier used to distinguish different configurations in the CI pipeline # type --> Specifies the Bokken agent type (e.g., Unity::VM, Unity::VM::osx, Unity::mobile::shield) @@ -25,9 +26,12 @@ small_agent_platform: flavor: b1.small -# PLATFORM CONFIGURATIONS----------------------------------------------------------------------------------------------------- +# PLATFORM CONFIGURATIONS---------------------------------------------------------------------------------------------------------------- # test_platforms specifies platforms that project will/can be tested on. # For desktops we include Windows, Ubuntu and Mac, for mobiles we include Android and iOS, for consoles we include ps4, ps5, switch, xbox360 and xboxOne +# For mobile and console a split was required for build and test jobs and this is also applied for desktops for consistency (though for desktop this split is not required) + + test_platforms: # Default platform is used for all basic jobs. Ubuntu was chosen since it's fastest and most available default: @@ -144,6 +148,8 @@ test_platforms: # EDITOR CONFIGURATIONS------------------------------------------------------------------------------- # Editors to be used for testing. NGOv2.X official support started from 6000.0 editor +# TODO: When a new editor will be released it should be added to this list + validation_editors: default: - trunk @@ -154,15 +160,17 @@ validation_editors: # Scripting backends used by Standalone RunTimeTests--------------------------------------------------- + scripting_backends: - mono - il2cpp # PROJECTS CONFIGURATIONS------------------------------------------------------------------------------- -# Projects within the repository that will be tested. -# The default project should be used for general validation, code coverage and other tests where choice of project should not matter (we are interested in package) +# Projects within the repository that will be tested/build. +# The default project should be used for general validation, code coverage and other tests where choice of project should not matter (because we are interested in NGO package) # has_tests --> describes if projects contains any tests to run. + projects: default: - name: testproject diff --git a/.yamato/webgl-build.yml b/.yamato/webgl-build.yml index cc0a577f40..df17f307e2 100644 --- a/.yamato/webgl-build.yml +++ b/.yamato/webgl-build.yml @@ -3,13 +3,13 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible for validating a successful building of a player on WebGl standalone platform. - # This job will not execute any tests, we only test building process. + # This job WILL NOT execute any tests, we only validate the building process. # This is because WebGL runs in browser and for tests to be executed we would need to consider having a web server, browser automation and overall complex test setup which currently we don't have. # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations - # Jobs are generated using nested loops through: - # 1. For the default project (from project.metafile). In this case the default project (testptoject) is used as a context for the build. + # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations + # Jobs configurations are generated using nested loops through: + # 1. For the default project (project is used only as a context for the build). # 2. For all desktop platforms (Windows, Ubuntu, macOS) # 3. For all supported Unity Editor versions (For NGOv2.X it means 6000+) @@ -18,12 +18,12 @@ # The jobs name being generated would be for example "WebGl Build - testproject [win, 6000.0, il2cpp]" (name visible in Yamato) or "webgl_build_testproject_win_6000.0" (job name when querying for it) -# TECHNICAL CONSTRAINTS---------------------------------------------------------------- - # WebGL requires IL2CPP scripting backend (Mono not supported) +# TECHNICAL CONSIDERATIONS---------------------------------------------------------------- + # WebGL requires IL2CPP scripting backend (Mono is not supported) # We are not using ARM64 architectures since we only perform a build action. x64 architectures are preferred for build phase (in order to optimize available resource usage) # We only perform build validation (no runtime testing) -# QUALITY THOUGHTS-------------------------------------------------------------------- +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- # In the future we could try to implement an infrastructure to run test in webgl context but this could be quite complicated and would need to be evaluated if it's worth it # To see where this job is included (in trigger job definitions) look into _triggers.yml file @@ -39,17 +39,14 @@ webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor -c webgl -c il2cpp + - unity-downloader-cli -u {{ editor }} -c Editor -c webgl -c il2cpp # Downloading the editor with additional webgl and il2cpp components # The following step builds the player with defined options such as: # Suite parameter if defined since it's a mandatory field to define which test suite should be used, but it doesn't matter in this case since we won't run any tests (--suite) - # We specify build platform (--platform), scripting backend (--scripting-backend) and preinstalled editor location (--editor-location) - # We will be building project defined in (--testproject) - # Custom artifact paths for logs and build output (via --artifacts_path and --player-save-path) # Editor is run in batchmode, which means that Unity runs command line arguments without the need for human interaction. It also suppresses pop-up windows that require human interaction (such as the Save Scene window). We should always run Unity in batch mode when using command line arguments, because it allows automation to run without interruption. (--extra-editor-arg=-batchmode) # Engine is initialized in ‘nographics’ mode since we don't need any graphics for this case (--extra-editor-arg=-nographics) # In case of failure the job will be rerunned once (--reruncount=1) with clean library (--clean-library-on-rerun) - # This will perform only building phase (--build-only) with a default timeout of 3m (--timeout=1800) + # This will perform only building phase (--build-only) with a timeout of 3m (--timeout=1800) - UnifiedTestRunner --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: logs: From 5ccb25ab2e89a0b288e5d9b97d9901dfb370ea8e Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Wed, 12 Mar 2025 11:10:04 +0100 Subject: [PATCH 18/39] Added ci README file --- .yamato/CI_README.md | 80 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .yamato/CI_README.md diff --git a/.yamato/CI_README.md b/.yamato/CI_README.md new file mode 100644 index 0000000000..2e0a761714 --- /dev/null +++ b/.yamato/CI_README.md @@ -0,0 +1,80 @@ +# Netcode for GameObjects CI Documentation + +## Overview +This document provides an overview of the Continuous Integration (CI) implementation for Netcode for GameObjects. +Specifics of each test are described within related files (for example .yamato/package-tests.yml) and this file present high level overview related to overall implementation. + +## Test Configurations +CI related files are present inside .yamato/ folder and we can distinguish specific tests + +### Helper jobs +- `.yamato/package-pack.yml` responsible for generating package artifacts (.tgz) required for testing and publishing. +- `.yamato/project-pack.yml` responsible for generating package artifacts (.tgz) required for testing and publishing. This packs all packages of a given project. +- `.yamato/_run-all.yml` responsible for grouping tests into groups for easier management (for example "all console tests"). +- `.yamato/_triggers.yml` responsible for defining triggers (PR, nightly, weekly etc.) and defining which tests to run. +- `disable-burst-if-requested.py` responsible for helping to disable burst if needed. +- `.yamato/project.metafile` being the central configuration file defining test environments, platforms, projects, editors and other parameters used. + +### In-Editor Tests +- `.yamato/package-tests.yml` responsible for running package tests. +- `.yamato/project-tests.yml` responsible for running project tests. +- `.yamato/project-updated-dependencies-test.yml` responsible for running tests after updating dependencies to latest ones. +- `.yamato/project-standards.yml` responsible for running standards check on a package. +- `.yamato/code-coverage.yml` responsible for running code coverage analysis. +- `.yamato/performance-tests.yml` responsible for running performance tests. +- `.yamato/webgl-build.yml` responsible for making a build for webgl platform. + +### Standalone Tests +- `.yamato/desktop-standalone-tests.yml` responsible for running tests on standalone desktops. +- `.yamato/console-standalone-test.yml` responsible for running tests on standalone consoles. +- `.yamato/mobile-standalone-test.yml` responsible for running tests on standalone mobiles. + +### Wrench jobs +This file is auto generated by wrench and used for automating release process. +Those tests can also be used by our CI as dependencies. +- `.yamato/wrench/api-validation-jobs.yml` responsible for validating api for a given package version (for example if there are no breaking changes when updating patch version) +- `.yamato/wrench/package-pack-jobs.yml` responsible for generating package artifacts (.tgz) required for testing and publishing. +- `.yamato/wrench/preview-a-p-v.yml` responsible for validating PVP (Package Verification Profiles) standards for the package. +- `.yamato/wrench/promotion-jobs.yml` responsible for publishing. +- `.yamato/wrench/publish-trigger.yml` responsible for defining jobs that need to pass in order to publish. +- `.yamato/wrench/recipe-regeneration.yml` responsible for verifying regeneration of wrench recipes (no manual changes). +- `.yamato/wrench/validation-jobs.yml` responsible for running package tests. +- `.yamato/wrench/wrench_config.json` this is the configuration file for wrench. +- `.yamato/generated-scripts/` is a collection of infrastructure instability detection tools. + +## Jobs Configuration Structure +This section explains some configuration details that may be confusing while looking at .yml files. + +### Job Generation +Our configuration uses a dynamic job generation approach based on the `.yamato/project.metafile` configuration. While it may appear that only one job is defined per file, the system (Yamato) actually generates multiple jobs through nested loops. +Usually only 1 job is defined and then generated with multiple configurations/parameters but in case of standalone tests 2 jobs are defined which are responsible for building and then testing. + +### Job Naming Convention +Because the jobs are being generated their names will be filled by given parameters from project.metfaile. For example jobs from .yamato/console-standalone-test may have names like: +- Display Name (in Yamato): `Build testproject - [win, 6000.0, il2cpp]` +- Internal Job Name (used when adding dependency for example): `console_standalone_build_testproject_win_6000.0` + +## Platform Support +Currently, the CI implementation supports the following platforms: +1. Console Platforms: + - Switch + - PS4 + - PS5 + - Xbox360 + - XboxOne +2. Mobile Platforms: + - Android + - iOS +3. Desktop Platforms: + - Windows + - Ubuntu + - macOS +4. Unity Editor Versions + - Supports NGOv2.X (6000.0+ editors) +5. Architectures + - x64 + - ARM64 (This is present for consoles/mobiles but will be extended. More information is present in specific standalone test files) + +## Design Considerations +In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. +The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. \ No newline at end of file From 7ce8fdebb8ef5cc9f288649b0c5f2a4a612aa503 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Wed, 12 Mar 2025 11:16:41 +0100 Subject: [PATCH 19/39] Adjusted comments after creation of CI_README file --- .yamato/_run-all.yml | 9 ++------- .yamato/_triggers.yml | 4 ++++ .yamato/code-coverage.yml | 8 +------- .yamato/console-standalone-test.yml | 8 +------- .yamato/desktop-standalone-tests.yml | 8 +------- .yamato/mobile-standalone-test.yml | 6 ------ .yamato/package-pack.yml | 6 ------ .yamato/package-tests.yml | 6 ------ .yamato/performance-tests.yml | 6 ------ .yamato/project-pack.yml | 6 ------ .yamato/project-standards.yml | 6 ------ .yamato/project-tests.yml | 6 ------ .yamato/project-updated-dependencies-test.yml | 8 +------- .yamato/webgl-build.yml | 6 ------ 14 files changed, 10 insertions(+), 83 deletions(-) diff --git a/.yamato/_run-all.yml b/.yamato/_run-all.yml index ab056a94c5..10c1e0490d 100644 --- a/.yamato/_run-all.yml +++ b/.yamato/_run-all.yml @@ -5,13 +5,8 @@ # Those jobs group together related test as dependencies to allow to easily manage running a given set of tests. # This enables efficient test execution for different validation scenarios -# CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations - # Jobs are generated using nested loops through: - - # It's important to ensure that all generated in such way dependencies exist (this can be verified in Yamato) - # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. - # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. +# QUALITY CONSIDERATIONS--------------------------------------------------------------- + # It's important to ensure that all dependencies exist (this can be verified in Yamato) since a modification in parameters may result in a given job not being generated, and thus we will not be able to run such erroneous job. #----------------------------------------------------------------------------------- diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index 436d51787b..352f1f6c08 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -32,6 +32,10 @@ # Jobs configurations are generated by ensuring that all dependencies are successful. # The dependencies are taken from _run-all.yml file where we can gather multiple tests into proper sets +# QUALITY CONSIDERATIONS--------------------------------------------------------------- +# It's important to ensure that all dependencies exist (this can be verified in Yamato) since a modification in parameters may result in a given job not being generated, and thus we will not be able to run such erroneous job. + + #----------------------------------------------------------------------------------- diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index b2412f6c8e..08deb8c496 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -10,16 +10,10 @@ # Additional metrics for coverage analysis # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: # 1. For default platform only (Ubuntu) since coverage would not vary between platforms (no need for checks on more platforms) # 2. For default editor version (trunk) since coverage would not vary between editors (no need for checks on more editors) - - # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. - # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. - - # The jobs name being generated would be for example "Code Coverage - NGO [testproject, 6000.0]" (name visible in Yamato) or "code_coverage_testproject_6000.0" (job name when querying for it) - + #TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # In theory this job also runs package tests, but we don't want to use it as default since is heavier (because of added coverage analysis) and coverage is not changing that often # Requires Unity Editor installation diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index fe3b3c3ef6..8cf11a7b97 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -6,17 +6,11 @@ # Those tests cover both PlayMode and EditMode tests from package test assemblies. # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: # 1. For all console platform (Switch, ps4, ps5, xbox360, xboxOne) # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) ` # 3. For the default project. - - # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. - # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. - - # The jobs name being generated would be for example "Build testproject - [win, 6000.0, il2cpp]" (name visible in Yamato) or "console_standalone_build_testproject_win_6000.0" (job name when querying for it) - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # For console devices a split is required into two phases: # 1. Build Phase: Creates standalone players for console platforms diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 430aeefc37..f4983c7778 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -6,18 +6,12 @@ # Those tests cover both PlayMode and EditMode tests from package test assemblies. # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: # 1. For all desktop platform (Windows, macOS, Ubuntu) # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) # 3. For the default project. # 4. For all scripting backends (mono, il2cpp) - - # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. - # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. - - # The jobs name being generated would be for example "Standalone Build - NGO testproject - [win, mono, 6000.0]" (name visible in Yamato) or "desktop_standalone_build_testproject_win_mono_6000.0" (job name when querying for it) - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # For desktop devices a split is into two phases is not required but we use it for consistency with setup of others standalone platforms: # 1. Build Phase: Creates standalone players for desktop platforms diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index 20ca1e139d..cc1e19011c 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -6,17 +6,11 @@ # Those tests cover both PlayMode and EditMode tests from package test assemblies. # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: # 1. For all mobile platform (Android, iOS) # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) # 3. For the default project. - # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. - # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. - - # The jobs name being generated would be for example "Build testproject - [win, 6000.0, il2cpp]" (name visible in Yamato) or "build_testproject_win_6000.0" (job name when querying for it) - # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # For mobile devices a split is required into two phases: # 1. Build Phase: Creates standalone players for mobile platforms diff --git a/.yamato/package-pack.yml b/.yamato/package-pack.yml index fce9ba3987..7568bdddcb 100644 --- a/.yamato/package-pack.yml +++ b/.yamato/package-pack.yml @@ -8,15 +8,9 @@ # Because of that validation we can detect errors at the early stage of testing so not to waste CI resources # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: # 1. For all desktop platforms (Windows, Ubuntu, macOS) - # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. - # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. - - # The jobs name being generated would be for example "Package Pack (and x-ray) - NGO [win]" (name visible in Yamato) or "package_pack_-_ngo_win" (job name when querying for it) - # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # Job does not require Unity Editor in order to perform packing. # Job uses PVP x-ray for lightweight validation diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index c9185b06ae..cdaf17a31d 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -7,16 +7,10 @@ # Additionally it combines Package Verification Pipeline (PVP) validation. This ensures that package is compatible with Unity standards # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs are generated using nested loops through: # 1. For all desktop platforms (Windows, Ubuntu, macOS) # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) - # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. - # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. - - # The jobs name being generated would be for example "Package Test - NGO testproject [win, 6000.0]" (name visible in Yamato) or "package_test_-_ngo_6000.0_win" (job name when querying for it) - # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # This job runs in Editor context only (no player builds required) # No scripting backend variations needed (Editor context) diff --git a/.yamato/performance-tests.yml b/.yamato/performance-tests.yml index fd0df85a34..a9cbd1a979 100644 --- a/.yamato/performance-tests.yml +++ b/.yamato/performance-tests.yml @@ -7,17 +7,11 @@ # Currently because of lack of performance tests this job will always return "no tests have been selected" and because oif that it's not included in any trigger jobs. # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs configurations are generated using nested loops through: # 1. For all desktop platforms (Windows, Ubuntu, macOS) # 2. For all supported Unity Editor versions (For NGOv2.X it means 6000+) # 3. For the default project (project is used only as a context for the build). TODO-comment: if performance tests would be included in projects then we should make an approperiate split. - # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. - # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. - - # The jobs name being generated would be for example "Performance editor Tests - NGO testproject [win, 6000.0] (No Data Reporting)" (name visible in Yamato) or "performance_editor_tests_-_NGO_testproject_6000.0_no_data_reporting" (job name when querying for it) - # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # Tests are run in Editor context only # No performance metrics are reported to monitoring systems diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml index f2f3d2f841..bbcbc6380b 100644 --- a/.yamato/project-pack.yml +++ b/.yamato/project-pack.yml @@ -6,16 +6,10 @@ # The job itself doesn't test anything specific but rather it prepares project packages that will be consumed by other pipeline jobs. # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs configurations are generated using nested loops through: # 1. For all projects (testproject, minimalproject, testproject-tools-integration). # 2. For all desktop platforms (Win, Ubuntu, Mac) - # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. - # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. - - # The jobs name being generated would be for example "Project Pack - testproject [win]" (name visible in Yamato) or "project_pack_-_testproject_win" (job name when querying for it) - # TECHNICAL CONSIDERATIONS-------------------------------------------------------------------- # Job does not require Unity Editor in order to perform packing. # In theory, we could just use one platform for packing projects (for example ubuntu) but in order to reduce confusion we are using same platform as the job utilizing project pack as dependency. diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index 15798cc314..7334746eba 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -10,17 +10,11 @@ # Solution file synchronization # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs configurations are generated using nested loops through: # 1. For all NGO projects (testproject, testproject-tools-interation, minimalproject) # 2. For default platform only (Ubuntu) since standards would not vary between platforms (no need for checks on more platforms) # 3. For default editor version (trunk) since standards would not vary between editors (no need for checks on more editors) - # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. - # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. - - # The jobs name being generated would be for example "Standards Check - NGO testproject [win, 6000.0]" (name visible in Yamato) or "standards_win_testproject_6000.0" (job name when querying for it) - # TECHNICAL CONSTRAINTS--------------------------------------------------------------- # Requires .NET SDK installed (should be preinstalled on the image so we just check for version) # Needs Unity Editor for solution synchronization diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 5fc3ce6a6e..63b3bc6226 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -7,17 +7,11 @@ # NGO package tests are NOT being executed within this job (those are handled in separate package test jobs) # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs configurations are generated using nested loops through: # 1. For all projects WITH TESTS (filtered by has_tests flag) (testproject, testproject-tools-interation) [For more info look into project.metafile configuration] # 2. For all desktop platforms (Windows, Ubuntu, macOS) # 3. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) - # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. - # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. - - # The jobs name being generated would be for example "Project Test - NGO testproject [win, 6000.0]" (name visible in Yamato) or "test_testproject_win_6000.0" (job name when querying for it) - # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # This job runs in Editor context only (no player builds is required) # No scripting backend variations needed (Editor context) diff --git a/.yamato/project-updated-dependencies-test.yml b/.yamato/project-updated-dependencies-test.yml index 80db9a15ba..f2e7b06877 100644 --- a/.yamato/project-updated-dependencies-test.yml +++ b/.yamato/project-updated-dependencies-test.yml @@ -6,17 +6,11 @@ # This helps detect potential breaking changes from dependency updates early # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs configurations are generated using nested loops through: # 1. For all projects (testproject, minimalproject, testproject-tools-integration). # 2. For all desktop platforms (Win, Ubuntu, Mac) # 3. For all supported editors (For NGOv2.X it means 6000+) - - # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. - # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. - - # The jobs name being generated would be for example "Updated Dependencies Test - NGO testproject [win, 6000.0]" (name visible in Yamato) or "updated-dependencies_testproject_NGO_win_6000.0" (job name when querying for it) - + # TECHNICAL CONSIDERATIONS---------------------------------------------------------------- # This job requires successful project packaging before execution (job dependency) # This job tests only NGO package dependencies (com.unity.netcode.gameobjects) diff --git a/.yamato/webgl-build.yml b/.yamato/webgl-build.yml index df17f307e2..e95cfff3ac 100644 --- a/.yamato/webgl-build.yml +++ b/.yamato/webgl-build.yml @@ -7,17 +7,11 @@ # This is because WebGL runs in browser and for tests to be executed we would need to consider having a web server, browser automation and overall complex test setup which currently we don't have. # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations # Jobs configurations are generated using nested loops through: # 1. For the default project (project is used only as a context for the build). # 2. For all desktop platforms (Windows, Ubuntu, macOS) # 3. For all supported Unity Editor versions (For NGOv2.X it means 6000+) - # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. - # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. - - # The jobs name being generated would be for example "WebGl Build - testproject [win, 6000.0, il2cpp]" (name visible in Yamato) or "webgl_build_testproject_win_6000.0" (job name when querying for it) - # TECHNICAL CONSIDERATIONS---------------------------------------------------------------- # WebGL requires IL2CPP scripting backend (Mono is not supported) # We are not using ARM64 architectures since we only perform a build action. x64 architectures are preferred for build phase (in order to optimize available resource usage) From 6fdc9c387f44a7dd4a7d6170a16ee23efb5fe191 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Wed, 12 Mar 2025 12:34:54 +0100 Subject: [PATCH 20/39] corrected linter typo --- .yamato/console-standalone-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index 8cf11a7b97..d55eb2910b 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -9,7 +9,7 @@ # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: # 1. For all console platform (Switch, ps4, ps5, xbox360, xboxOne) # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) - ` # 3. For the default project. + # 3. For the default project. # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # For console devices a split is required into two phases: From 7cbeabf13be1af26a5f8ddb6ffd586ddd900ed0e Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Wed, 12 Mar 2025 13:03:06 +0100 Subject: [PATCH 21/39] Corrected job names --- .yamato/_run-all.yml | 8 ++++---- .yamato/console-standalone-test.yml | 2 +- .yamato/mobile-standalone-test.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.yamato/_run-all.yml b/.yamato/_run-all.yml index 10c1e0490d..985e1d2f30 100644 --- a/.yamato/_run-all.yml +++ b/.yamato/_run-all.yml @@ -137,7 +137,7 @@ run_all_project_tests_mobile_standalone: {% for project in projects.default -%} {% for platform in test_platforms.mobile_test -%} {% for editor in validation_editors.all -%} - - .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} + - .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} @@ -150,7 +150,7 @@ run_all_project_tests_mobile_standalone_trunk: {% for project in projects.default -%} {% for platform in test_platforms.mobile_test -%} {% for editor in validation_editors.default -%} - - .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} + - .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} @@ -163,7 +163,7 @@ run_all_project_tests_console_standalone: {% for project in projects.default -%} {% for platform in test_platforms.console_test -%} {% for editor in validation_editors.all -%} - - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} + - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} @@ -176,7 +176,7 @@ run_all_project_tests_console_standalone_trunk: {% for project in projects.default -%} {% for platform in test_platforms.console_test -%} {% for editor in validation_editors.default -%} - - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} + - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index d55eb2910b..3cfa324657 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -76,7 +76,7 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% for project in projects.default -%} {% for platform in test_platforms.console_test -%} {% for editor in validation_editors.all -%} -console_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: +console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: name: Run {{ project.name }} Tests - [{{ platform.name }}, {{ editor }}, il2cpp] agent: type: {{ platform.type }} diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index cc1e19011c..c35c73e401 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -64,7 +64,7 @@ mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% for project in projects.default -%} {% for platform in test_platforms.mobile_test -%} {% for editor in validation_editors.all -%} -mobile_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: +mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: name: Run {{ project.name }} Tests - [{{ platform.name }}, {{ editor }}, il2cpp] agent: type: {{ platform.type }} From b814105f483dd5c7917faf2677b54edb60056d10 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Wed, 12 Mar 2025 16:25:51 +0100 Subject: [PATCH 22/39] Readme file name correction --- .yamato/{CI_README.md => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .yamato/{CI_README.md => README.md} (100%) diff --git a/.yamato/CI_README.md b/.yamato/README.md similarity index 100% rename from .yamato/CI_README.md rename to .yamato/README.md From 9b20c8621a725d0197c7396051cb3d7b0892131c Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Wed, 12 Mar 2025 17:26:34 +0100 Subject: [PATCH 23/39] Added wait flag when downloading editor --- .yamato/code-coverage.yml | 2 +- .yamato/console-standalone-test.yml | 4 ++-- .yamato/desktop-standalone-tests.yml | 4 ++-- .yamato/mobile-standalone-test.yml | 4 ++-- .yamato/package-tests.yml | 2 +- .yamato/performance-tests.yml | 2 +- .yamato/project-standards.yml | 2 +- .yamato/project-tests.yml | 2 +- .yamato/webgl-build.yml | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 08deb8c496..6094215dc1 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -35,7 +35,7 @@ code_coverage_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor + - unity-downloader-cli -u {{ editor }} -c Editor --wait - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage -coverage-results-path=$PWD/test-results/CodeCoverage --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results artifacts: diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index 3cfa324657..35ef05e1ce 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -47,7 +47,7 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} + - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --wait - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 variables: # PS4 related @@ -86,7 +86,7 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: model: {{ platform.model }} {% endif %} commands: - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} + - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --wait - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 variables: # PS4 related diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index a47400cb0e..91aed3f600 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -40,7 +40,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} + - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} --wait - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: @@ -81,7 +81,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ {% endif %} commands: - - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} + - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} --wait - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 # If ubuntu, run rust echo server diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index c35c73e401..1f85f3b5d3 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -46,7 +46,7 @@ mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} + - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} --wait - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: @@ -75,7 +75,7 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: # Installing editor. We still need the editor to run tests on standalone build and for that the Editor is required - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} + - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} --wait {% if platform.standalone == "Android" %} # Download standalone UnityTestRunner and ADB setup diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index cdaf17a31d..f47c9a95c6 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -36,7 +36,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: XRAY_PROFILE: "supported ./pvpExceptions.json" UNITY_EXT_LOGGING: 1 commands: - - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait + - unity-downloader-cli -u {{ editor }} -c Editor --wait # Validate PVP checks for package. - upm-pvp test --unity .Editor --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --results pvp-results diff --git a/.yamato/performance-tests.yml b/.yamato/performance-tests.yml index a9cbd1a979..50443f2419 100644 --- a/.yamato/performance-tests.yml +++ b/.yamato/performance-tests.yml @@ -32,7 +32,7 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor # Installing basic editor + - unity-downloader-cli -u {{ editor }} -c Editor --wait # Installing basic editor - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata # TODO: when performance tests will be present we need to add actuall execution of this test artifacts: diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index 7334746eba..e1ed7590bb 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -42,7 +42,7 @@ standards_{{ platform.name }}_{{ project.name }}_{{ editor }}: - dotnet --version - dotnet format --version - - unity-downloader-cli -u {{ editor }} -c editor # Downloads basic editor + - unity-downloader-cli -u {{ editor }} -c editor --wait # Downloads basic editor - .Editor/Unity -batchmode -nographics -logFile - -executeMethod Packages.Rider.Editor.RiderScriptEditor.SyncSolution -projectPath {{ project.path }} -quit # This command is used to invoke Unity in a "headless" mode. It's used to sync the project - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check # Runs standards check {% endfor -%} diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 63b3bc6226..9b44925d7c 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -36,7 +36,7 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: flavor: {{ platform.flavor }} commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - - unity-downloader-cli -u {{ editor }} -c Editor # Installing basic editor for tests execution + - unity-downloader-cli -u {{ editor }} -c Editor --wait # Installing basic editor for tests execution - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--reruncount=1 --clean-library-on-rerun" # project tests execution via upm-ci artifacts: logs: diff --git a/.yamato/webgl-build.yml b/.yamato/webgl-build.yml index e95cfff3ac..42c70c4952 100644 --- a/.yamato/webgl-build.yml +++ b/.yamato/webgl-build.yml @@ -33,7 +33,7 @@ webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor -c webgl -c il2cpp # Downloading the editor with additional webgl and il2cpp components + - unity-downloader-cli -u {{ editor }} -c Editor -c webgl -c il2cpp --wait # Downloading the editor with additional webgl and il2cpp components # The following step builds the player with defined options such as: # Suite parameter if defined since it's a mandatory field to define which test suite should be used, but it doesn't matter in this case since we won't run any tests (--suite) From 93baccdab3ea6905999bff874b5a5a186e1b3c16 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Wed, 12 Mar 2025 19:13:40 +0100 Subject: [PATCH 24/39] Added comment --- .yamato/desktop-standalone-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 91aed3f600..10a4d0d966 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -71,7 +71,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ image: {{ platform.image }} flavor: {{ platform.flavor }} -# Set additional variables for running the echo server +# Set additional variables for running the echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority) {% if platform.name != "win" %} variables: ECHO_SERVER_PORT: "7788" @@ -84,7 +84,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} --wait - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 -# If ubuntu, run rust echo server +# If ubuntu, run rust echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority) {% if platform.name != "win" %} - git clone https://github.com/Unity-Technologies/mps-common-multiplayer-backend.git # Install rust From b0504182be1b6c2991e671729dfa210fc3ae14f2 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Thu, 13 Mar 2025 12:44:23 +0100 Subject: [PATCH 25/39] Modified mac model to Silicon --- .yamato/desktop-standalone-tests.yml | 6 ++++++ .yamato/project.metafile | 1 + 2 files changed, 7 insertions(+) diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 10a4d0d966..6a5c5bf796 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -39,6 +39,9 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ type: {% if platform.name == "mac" %} {{ platform.type }} {% else %} {{ platform.type }}::GPU {% endif %} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} --wait - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 @@ -70,6 +73,9 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ type: {% if platform.name == "mac" %} {{ platform.type }} {% else %} {{ platform.type }}::GPU {% endif %} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} # Set additional variables for running the echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority) {% if platform.name != "win" %} diff --git a/.yamato/project.metafile b/.yamato/project.metafile index 65cab054eb..5a3f1b2b44 100644 --- a/.yamato/project.metafile +++ b/.yamato/project.metafile @@ -58,6 +58,7 @@ test_platforms: image: package-ci/macos-13:v4 flavor: m1.mac standalone: StandaloneOSX + model: M1 # The default model (an x64 Intel Mac VM) quite often caused a known issue of doing all the bitflips in packages resulting in failures # For mobile devices there is a split between the build and run phase so there is a need of splitting specification for both # TODO: For android we could consider adding ARM64 configuration mobile_build: From 0d1904ffdfed7934e728c11d8bdf89a3b886fad3 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Thu, 13 Mar 2025 12:57:37 +0100 Subject: [PATCH 26/39] Corrected editor download command --- .yamato/code-coverage.yml | 2 +- .yamato/console-standalone-test.yml | 4 ++-- .yamato/desktop-standalone-tests.yml | 4 ++-- .yamato/mobile-standalone-test.yml | 4 ++-- .yamato/package-tests.yml | 2 +- .yamato/project-standards.yml | 2 +- .yamato/project-tests.yml | 2 +- .yamato/webgl-build.yml | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 6094215dc1..102b03c5f6 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -35,7 +35,7 @@ code_coverage_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor --wait + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage -coverage-results-path=$PWD/test-results/CodeCoverage --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results artifacts: diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index 35ef05e1ce..98c820c3f4 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -47,7 +47,7 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --wait + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 variables: # PS4 related @@ -86,7 +86,7 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: model: {{ platform.model }} {% endif %} commands: - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --wait + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 variables: # PS4 related diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 6a5c5bf796..7e5fff01b3 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -43,7 +43,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} --wait + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: @@ -87,7 +87,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ {% endif %} commands: - - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} --wait + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 # If ubuntu, run rust echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority) diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index 1f85f3b5d3..89e7c52c6c 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -46,7 +46,7 @@ mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} --wait + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: @@ -75,7 +75,7 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: # Installing editor. We still need the editor to run tests on standalone build and for that the Editor is required - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} --wait + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} {% if platform.standalone == "Android" %} # Download standalone UnityTestRunner and ADB setup diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index f47c9a95c6..96900f4026 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -36,7 +36,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: XRAY_PROFILE: "supported ./pvpExceptions.json" UNITY_EXT_LOGGING: 1 commands: - - unity-downloader-cli -u {{ editor }} -c Editor --wait + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor # Validate PVP checks for package. - upm-pvp test --unity .Editor --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --results pvp-results diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index e1ed7590bb..513b51f6c6 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -42,7 +42,7 @@ standards_{{ platform.name }}_{{ project.name }}_{{ editor }}: - dotnet --version - dotnet format --version - - unity-downloader-cli -u {{ editor }} -c editor --wait # Downloads basic editor + - unity-downloader-cli --fast --wait -u {{ editor }} -c editor # Downloads basic editor - .Editor/Unity -batchmode -nographics -logFile - -executeMethod Packages.Rider.Editor.RiderScriptEditor.SyncSolution -projectPath {{ project.path }} -quit # This command is used to invoke Unity in a "headless" mode. It's used to sync the project - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check # Runs standards check {% endfor -%} diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 9b44925d7c..14cddc79d4 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -36,7 +36,7 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: flavor: {{ platform.flavor }} commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - - unity-downloader-cli -u {{ editor }} -c Editor --wait # Installing basic editor for tests execution + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor # Installing basic editor for tests execution - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--reruncount=1 --clean-library-on-rerun" # project tests execution via upm-ci artifacts: logs: diff --git a/.yamato/webgl-build.yml b/.yamato/webgl-build.yml index 42c70c4952..717b9538c9 100644 --- a/.yamato/webgl-build.yml +++ b/.yamato/webgl-build.yml @@ -33,7 +33,7 @@ webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} commands: - - unity-downloader-cli -u {{ editor }} -c Editor -c webgl -c il2cpp --wait # Downloading the editor with additional webgl and il2cpp components + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c webgl -c il2cpp # Downloading the editor with additional webgl and il2cpp components # The following step builds the player with defined options such as: # Suite parameter if defined since it's a mandatory field to define which test suite should be used, but it doesn't matter in this case since we won't run any tests (--suite) From 37638177a7f4263319eda1709169f4f76ee38546 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Thu, 13 Mar 2025 13:06:00 +0100 Subject: [PATCH 27/39] Updated macOS image to support M1 model --- .yamato/project.metafile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/project.metafile b/.yamato/project.metafile index 5a3f1b2b44..a04afbc533 100644 --- a/.yamato/project.metafile +++ b/.yamato/project.metafile @@ -55,7 +55,7 @@ test_platforms: model: rtx2080 - name: mac type: Unity::VM::osx - image: package-ci/macos-13:v4 + image: package-ci/macos-13-arm64:v4 # ARM64 to support M1 model (below) flavor: m1.mac standalone: StandaloneOSX model: M1 # The default model (an x64 Intel Mac VM) quite often caused a known issue of doing all the bitflips in packages resulting in failures From 5ef17a1b6fcc2a40fcf6b65b40880ee7711b5b1c Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Thu, 13 Mar 2025 13:11:58 +0100 Subject: [PATCH 28/39] Added --arch64 param --- .yamato/desktop-standalone-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 7e5fff01b3..11852bf06d 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -87,7 +87,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ {% endif %} commands: - - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 # If ubuntu, run rust echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority) From 001934ee36cb82ad5ff8721812e70fbabf63bdff Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Thu, 13 Mar 2025 13:20:32 +0100 Subject: [PATCH 29/39] Added models considerations to CI configurations --- .yamato/code-coverage.yml | 3 +++ .yamato/console-standalone-test.yml | 5 ++++- .yamato/mobile-standalone-test.yml | 5 ++++- .yamato/package-pack.yml | 3 +++ .yamato/package-tests.yml | 3 +++ .yamato/performance-tests.yml | 3 +++ .yamato/project-pack.yml | 3 +++ .yamato/project-standards.yml | 3 +++ .yamato/project-tests.yml | 3 +++ .yamato/project-updated-dependencies-test.yml | 3 +++ .yamato/webgl-build.yml | 3 +++ 11 files changed, 35 insertions(+), 2 deletions(-) diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 102b03c5f6..4d75d5b7e2 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -34,6 +34,9 @@ code_coverage_{{ platform.name }}_{{ editor }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index 98c820c3f4..ee74bfee0b 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -46,6 +46,9 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 @@ -83,7 +86,7 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} {% if platform.model %} - model: {{ platform.model }} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index 89e7c52c6c..8169d34e8d 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -45,6 +45,9 @@ mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 @@ -71,7 +74,7 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: image: {{ platform.image }} flavor: {{ platform.flavor }} {% if platform.model %} - model: {{ platform.model }} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: # Installing editor. We still need the editor to run tests on standalone build and for that the Editor is required diff --git a/.yamato/package-pack.yml b/.yamato/package-pack.yml index 7568bdddcb..f4b02dc19a 100644 --- a/.yamato/package-pack.yml +++ b/.yamato/package-pack.yml @@ -30,6 +30,9 @@ package_pack_-_ngo_{{ platform.name }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} timeout: 0.25 variables: XRAY_PROFILE: "supported ./pvpExceptions.json" diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index 96900f4026..a21812d420 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -32,6 +32,9 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} variables: XRAY_PROFILE: "supported ./pvpExceptions.json" UNITY_EXT_LOGGING: 1 diff --git a/.yamato/performance-tests.yml b/.yamato/performance-tests.yml index 50443f2419..52eae2cb47 100644 --- a/.yamato/performance-tests.yml +++ b/.yamato/performance-tests.yml @@ -31,6 +31,9 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - unity-downloader-cli -u {{ editor }} -c Editor --wait # Installing basic editor - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml index bbcbc6380b..5a0337a6a1 100644 --- a/.yamato/project-pack.yml +++ b/.yamato/project-pack.yml @@ -29,6 +29,9 @@ project_pack_-_{{ project.name }}_{{ platform.name }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - upm-ci project pack --project-path {{ project.path }} diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index 513b51f6c6..aaec242690 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -37,6 +37,9 @@ standards_{{ platform.name }}_{{ project.name }}_{{ editor }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: # .NET environment setup. Ensures required .NET SDK and formatting tools are available - dotnet --version diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 14cddc79d4..51ffcb499c 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -34,6 +34,9 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor # Installing basic editor for tests execution diff --git a/.yamato/project-updated-dependencies-test.yml b/.yamato/project-updated-dependencies-test.yml index f2e7b06877..19ae953e27 100644 --- a/.yamato/project-updated-dependencies-test.yml +++ b/.yamato/project-updated-dependencies-test.yml @@ -31,6 +31,9 @@ updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type updated-dependencies-tests --package-filter com.unity.netcode.gameobjects diff --git a/.yamato/webgl-build.yml b/.yamato/webgl-build.yml index 717b9538c9..aea415562b 100644 --- a/.yamato/webgl-build.yml +++ b/.yamato/webgl-build.yml @@ -32,6 +32,9 @@ webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c webgl -c il2cpp # Downloading the editor with additional webgl and il2cpp components From 3c7d3565bcaa4c16d1dd3ad3f5254c448812c374 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Thu, 13 Mar 2025 13:32:52 +0100 Subject: [PATCH 30/39] Added arm64 component to editors where necessary --- .yamato/code-coverage.yml | 2 +- .yamato/desktop-standalone-tests.yml | 2 +- .yamato/package-tests.yml | 2 +- .yamato/performance-tests.yml | 2 +- .yamato/project-standards.yml | 2 +- .yamato/project-tests.yml | 2 +- .yamato/webgl-build.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 4d75d5b7e2..ced6277452 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -38,7 +38,7 @@ code_coverage_{{ platform.name }}_{{ editor }}: model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage -coverage-results-path=$PWD/test-results/CodeCoverage --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results artifacts: diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 11852bf06d..6611d84d97 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -43,7 +43,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index a21812d420..722ea368df 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -39,7 +39,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: XRAY_PROFILE: "supported ./pvpExceptions.json" UNITY_EXT_LOGGING: 1 commands: - - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models # Validate PVP checks for package. - upm-pvp test --unity .Editor --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --results pvp-results diff --git a/.yamato/performance-tests.yml b/.yamato/performance-tests.yml index 52eae2cb47..26bf8102f4 100644 --- a/.yamato/performance-tests.yml +++ b/.yamato/performance-tests.yml @@ -35,7 +35,7 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - - unity-downloader-cli -u {{ editor }} -c Editor --wait # Installing basic editor + - unity-downloader-cli -u {{ editor }} -c Editor --wait {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata # TODO: when performance tests will be present we need to add actuall execution of this test artifacts: diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index aaec242690..6a70daf149 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -45,7 +45,7 @@ standards_{{ platform.name }}_{{ project.name }}_{{ editor }}: - dotnet --version - dotnet format --version - - unity-downloader-cli --fast --wait -u {{ editor }} -c editor # Downloads basic editor + - unity-downloader-cli --fast --wait -u {{ editor }} -c editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Downloads basic editor - .Editor/Unity -batchmode -nographics -logFile - -executeMethod Packages.Rider.Editor.RiderScriptEditor.SyncSolution -projectPath {{ project.path }} -quit # This command is used to invoke Unity in a "headless" mode. It's used to sync the project - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check # Runs standards check {% endfor -%} diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 51ffcb499c..a39f957480 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -39,7 +39,7 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor # Installing basic editor for tests execution + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--reruncount=1 --clean-library-on-rerun" # project tests execution via upm-ci artifacts: logs: diff --git a/.yamato/webgl-build.yml b/.yamato/webgl-build.yml index aea415562b..393998f407 100644 --- a/.yamato/webgl-build.yml +++ b/.yamato/webgl-build.yml @@ -36,7 +36,7 @@ webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c webgl -c il2cpp # Downloading the editor with additional webgl and il2cpp components + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c webgl -c il2cpp {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Downloading the editor with additional webgl and il2cpp components # The following step builds the player with defined options such as: # Suite parameter if defined since it's a mandatory field to define which test suite should be used, but it doesn't matter in this case since we won't run any tests (--suite) From a463a49aa284242ee682b4d341ea649bc9fe637a Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Thu, 13 Mar 2025 14:24:14 +0100 Subject: [PATCH 31/39] Corrected editor architecture --- .yamato/desktop-standalone-tests.yml | 2 +- .yamato/package-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 6611d84d97..11852bf06d 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -43,7 +43,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index 722ea368df..a21812d420 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -39,7 +39,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: XRAY_PROFILE: "supported ./pvpExceptions.json" UNITY_EXT_LOGGING: 1 commands: - - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor # Validate PVP checks for package. - upm-pvp test --unity .Editor --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --results pvp-results From c82660435e94737ed7e4c0dc538a6d7d465f06f3 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Fri, 14 Mar 2025 10:46:46 +0100 Subject: [PATCH 32/39] corrected package-test with usage of arm64 for mac machines --- .yamato/package-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index a21812d420..2d304ad02e 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -39,7 +39,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: XRAY_PROFILE: "supported ./pvpExceptions.json" UNITY_EXT_LOGGING: 1 commands: - - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. # Validate PVP checks for package. - upm-pvp test --unity .Editor --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --results pvp-results From 4fe48d9b2df213e4668e14b85f4c6153d1164081 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Fri, 14 Mar 2025 14:11:16 +0100 Subject: [PATCH 33/39] Disabled rust server for macOS platform --- .yamato/desktop-standalone-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 11852bf06d..e5288a7195 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -78,7 +78,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ {% endif %} # Set additional variables for running the echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority) -{% if platform.name != "win" %} +{% if platform.name == "ubuntu" %} # Issues with win and mac are tracked in MTT-11606 variables: ECHO_SERVER_PORT: "7788" # Set this to ensure the DA codec tests will fail if they cannot connect to the echo-server @@ -91,7 +91,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 # If ubuntu, run rust echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority) -{% if platform.name != "win" %} +{% if platform.name == "ubuntu" %} # Issues with win and mac are tracked in MTT-11606 - git clone https://github.com/Unity-Technologies/mps-common-multiplayer-backend.git # Install rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y From 1eca439e348476a27e5e781dd726b309faefe6d3 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Fri, 14 Mar 2025 17:04:22 +0100 Subject: [PATCH 34/39] Updated model handling in desktop standalone tests --- .yamato/desktop-standalone-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index e5288a7195..302caf8079 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -73,12 +73,12 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ type: {% if platform.name == "mac" %} {{ platform.type }} {% else %} {{ platform.type }}::GPU {% endif %} image: {{ platform.image }} flavor: {{ platform.flavor }} -{% if platform.model %} - model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% if platform.name == mac %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile). In this case it's specifically for macOS (if used for win/ubuntu then it will cause rust server to fail connecting) {% endif %} # Set additional variables for running the echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority) -{% if platform.name == "ubuntu" %} # Issues with win and mac are tracked in MTT-11606 +{% if platform.name != "win" %} # Issues with win and mac are tracked in MTT-11606 variables: ECHO_SERVER_PORT: "7788" # Set this to ensure the DA codec tests will fail if they cannot connect to the echo-server @@ -91,7 +91,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 # If ubuntu, run rust echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority) -{% if platform.name == "ubuntu" %} # Issues with win and mac are tracked in MTT-11606 +{% if platform.name != "win" %} # Issues with win and mac are tracked in MTT-11606 - git clone https://github.com/Unity-Technologies/mps-common-multiplayer-backend.git # Install rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y From de4042b6cf9f6fae62ddd04252a0146a300b7afc Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Sat, 15 Mar 2025 21:20:00 +0100 Subject: [PATCH 35/39] Models correction --- .yamato/desktop-standalone-tests.yml | 2 +- .yamato/project.metafile | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 302caf8079..5d76a38917 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -73,7 +73,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ type: {% if platform.name == "mac" %} {{ platform.type }} {% else %} {{ platform.type }}::GPU {% endif %} image: {{ platform.image }} flavor: {{ platform.flavor }} -{% if platform.name == mac %} +{% if platform.model %} model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile). In this case it's specifically for macOS (if used for win/ubuntu then it will cause rust server to fail connecting) {% endif %} diff --git a/.yamato/project.metafile b/.yamato/project.metafile index a04afbc533..dedb946f05 100644 --- a/.yamato/project.metafile +++ b/.yamato/project.metafile @@ -46,13 +46,11 @@ test_platforms: image: package-ci/ubuntu-22.04:v4 flavor: b1.large standalone: StandaloneLinux64 - model: rtx2080 - name: win type: Unity::VM image: package-ci/win10:v4 flavor: b1.large standalone: StandaloneWindows64 - model: rtx2080 - name: mac type: Unity::VM::osx image: package-ci/macos-13-arm64:v4 # ARM64 to support M1 model (below) From b2800cc53fbc704c90d993a2ab308c2c27782093 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Mon, 17 Mar 2025 12:34:53 +0100 Subject: [PATCH 36/39] Revert "Models correction" This reverts commit de4042b6cf9f6fae62ddd04252a0146a300b7afc. --- .yamato/desktop-standalone-tests.yml | 2 +- .yamato/project.metafile | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 5d76a38917..302caf8079 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -73,7 +73,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ type: {% if platform.name == "mac" %} {{ platform.type }} {% else %} {{ platform.type }}::GPU {% endif %} image: {{ platform.image }} flavor: {{ platform.flavor }} -{% if platform.model %} +{% if platform.name == mac %} model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile). In this case it's specifically for macOS (if used for win/ubuntu then it will cause rust server to fail connecting) {% endif %} diff --git a/.yamato/project.metafile b/.yamato/project.metafile index dedb946f05..a04afbc533 100644 --- a/.yamato/project.metafile +++ b/.yamato/project.metafile @@ -46,11 +46,13 @@ test_platforms: image: package-ci/ubuntu-22.04:v4 flavor: b1.large standalone: StandaloneLinux64 + model: rtx2080 - name: win type: Unity::VM image: package-ci/win10:v4 flavor: b1.large standalone: StandaloneWindows64 + model: rtx2080 - name: mac type: Unity::VM::osx image: package-ci/macos-13-arm64:v4 # ARM64 to support M1 model (below) From efb5fd870de57cab30b9b688adde676dc71425d7 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Mon, 17 Mar 2025 12:38:20 +0100 Subject: [PATCH 37/39] Changed standalone for rust server --- .yamato/desktop-standalone-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 302caf8079..9f7ae04c79 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -78,7 +78,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ {% endif %} # Set additional variables for running the echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority) -{% if platform.name != "win" %} # Issues with win and mac are tracked in MTT-11606 +{% if platform.name == "ubuntu" %} # Issues with win and mac are tracked in MTT-11606 variables: ECHO_SERVER_PORT: "7788" # Set this to ensure the DA codec tests will fail if they cannot connect to the echo-server @@ -91,7 +91,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 # If ubuntu, run rust echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority) -{% if platform.name != "win" %} # Issues with win and mac are tracked in MTT-11606 +{% if platform.name == "ubuntu" %} # Issues with win and mac are tracked in MTT-11606 - git clone https://github.com/Unity-Technologies/mps-common-multiplayer-backend.git # Install rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y From fe016013e8274b56cd90bb8615c2b8e213fa7bf8 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Mon, 17 Mar 2025 13:01:04 +0100 Subject: [PATCH 38/39] Corrected if statement --- .yamato/desktop-standalone-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 9f7ae04c79..4ea2a29707 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -39,7 +39,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ type: {% if platform.name == "mac" %} {{ platform.type }} {% else %} {{ platform.type }}::GPU {% endif %} image: {{ platform.image }} flavor: {{ platform.flavor }} -{% if platform.model %} +{% if platform.name == "mac" %} model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: @@ -73,7 +73,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ type: {% if platform.name == "mac" %} {{ platform.type }} {% else %} {{ platform.type }}::GPU {% endif %} image: {{ platform.image }} flavor: {{ platform.flavor }} -{% if platform.name == mac %} +{% if platform.name == "mac" %} model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile). In this case it's specifically for macOS (if used for win/ubuntu then it will cause rust server to fail connecting) {% endif %} From 06247408b3f040080e916bea338f48a33b831f34 Mon Sep 17 00:00:00 2001 From: michalChrobot Date: Mon, 17 Mar 2025 16:25:14 +0100 Subject: [PATCH 39/39] Setting up rust server before running tests so there is no connection issue --- .yamato/desktop-standalone-tests.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 4ea2a29707..b6539b1cbf 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -78,7 +78,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ {% endif %} # Set additional variables for running the echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority) -{% if platform.name == "ubuntu" %} # Issues with win and mac are tracked in MTT-11606 +{% if platform.name != "win" %} # Issues with win and mac are tracked in MTT-11606 variables: ECHO_SERVER_PORT: "7788" # Set this to ensure the DA codec tests will fail if they cannot connect to the echo-server @@ -87,11 +87,8 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ {% endif %} commands: - - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 - # If ubuntu, run rust echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority) -{% if platform.name == "ubuntu" %} # Issues with win and mac are tracked in MTT-11606 +{% if platform.name != "win" %} # Issues with win and mac are tracked in MTT-11606 - git clone https://github.com/Unity-Technologies/mps-common-multiplayer-backend.git # Install rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y @@ -101,6 +98,8 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ - cd ./mps-common-multiplayer-backend/runtime && $HOME/.cargo/bin/cargo run --example ngo_echo_server -- --port $ECHO_SERVER_PORT & {% endif %} + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 artifacts: logs: paths: