# Copyright (c) 2018, 2026, Oracle and/or its affiliates.
# Copyright (c) 2013, Regents of the University of California
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are
# permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of
# conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice, this list of
# conditions and the following disclaimer in the documentation and/or other materials provided
# with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import print_function
import ast
import contextlib
import datetime
import glob
import gzip
import itertools
import json
import os
import pathlib
import re
import signal
import shlex
import shutil
import subprocess
import sys
import time
from functools import wraps
from pathlib import Path
from textwrap import dedent, indent
from xml.sax.saxutils import escape
from typing import cast, Union, Literal, overload
import downstream_tests
import mx_graalpython_benchmark
import mx_urlrewrites
import tempfile
from argparse import ArgumentParser
from dataclasses import dataclass
import mx
import mx_util
import mx_gate
import mx_native
import mx_unittest
import mx_sdk
import mx_sdk_vm_ng
import mx_subst
import mx_truffle
import mx_graalpython_bisect
import mx_graalpython_graalos
import mx_graalpython_import
import mx_pominit
import mx_graalpython_python_benchmarks
# re-export custom mx project classes so they can be used from suite.py
from mx_cmake import CMakeNinjaProject #pylint: disable=unused-import
from mx_gate import Task
from mx_graalpython_bench_param import PATH_MESO
# re-export custom mx project classes, so they can be used from suite.py
from mx_sdk_vm_ng import StandaloneLicenses, ThinLauncherProject, LanguageLibraryProject, DynamicPOMDistribution, DeliverableStandaloneArchive # pylint: disable=unused-import
if not sys.modules.get("__main__"):
# workaround for pdb++
sys.modules["__main__"] = type(sys)("")
def get_boolean_env(name, default=False):
env = os.environ.get(name)
if env is None:
return default
return env.lower() in ('true', '1')
SUITE = cast(mx.SourceSuite, mx.suite('graalpython'))
SUITE_COMPILER = mx.suite("compiler", fatalIfMissing=False)
GRAALPY_ABI_VERSION = 'graalpy253'
GRAALPY_ABIFLAGS = os.environ.get('GRAALPY_ABIFLAGS', '')
if not re.fullmatch(r'[A-Za-z0-9_]*', GRAALPY_ABIFLAGS):
mx.abort('GRAALPY_ABIFLAGS may only contain ASCII letters, digits, and underscores')
IS_RELEASE = SUITE.is_release()
FULL_GRAAL_VERSION = SUITE.release_version()
GRAAL_VERSION = FULL_GRAAL_VERSION if IS_RELEASE else FULL_GRAAL_VERSION[:-len('-dev')]
GRAAL_VERSION_MAJ_MIN = ".".join(GRAAL_VERSION.split(".")[:2])
PYTHON_VERSION = SUITE.suiteDict[f'{SUITE.name}:pythonVersion']
PYTHON_VERSION_MAJ_MIN = ".".join(PYTHON_VERSION.split('.')[:2])
LATEST_JAVA_HOME = {"JAVA_HOME": os.environ.get("LATEST_JAVA_HOME", mx.get_jdk().home)}
RUNNING_ON_LATEST_JAVA = os.environ.get("LATEST_JAVA_HOME", os.environ.get("JAVA_HOME")) == mx.get_jdk().home
HAS_JEP_454 = mx.get_jdk().version >= mx.VersionSpec("22.0.0")
# this environment variable is used by some of our maven projects to build against the unreleased master version during development
os.environ["GRAALPY_VERSION"] = GRAAL_VERSION
MAIN_BRANCH = 'master'
GRAALPY_PGO_PROFILE_ARTIFACT_GROUP = "graalpy"
GRAALPY_PGO_PROFILE_ARTIFACT_PREFIX = "pgo-"
GRAALPYTHON_MAIN_CLASS = "com.oracle.graal.python.shell.GraalPythonMain"
SANDBOXED_OPTIONS = [
'--experimental-options',
'--python.PosixModuleBackend=java',
'--python.Sha3ModuleBackend=java',
'--python.CompressionModulesBackend=java',
'--python.PyExpatModuleBackend=java',
'--python.UnicodeCharacterDatabaseNativeFallback=false',
]
SUBPROCESS_HEAVY_TESTS = [
'test_entropy_subprocess',
'test_repl',
'test_reparse',
'test_venv',
'test_patched_pip',
'test_wheel',
'test_startup',
'cpyext/test_shutdown',
'cpyext/test_fatal_exit',
]
MULTI_CONTEXT_EXCLUSIONS = [
*SUBPROCESS_HEAVY_TESTS,
# The test deletes loaded bytecode files, which trips up later reparsing of roots in the shared cache
'test_load_bytecode_file',
]
# Allows disabling rebuild for some mx commands such as graalpytest
DISABLE_REBUILD = get_boolean_env('GRAALPYTHON_MX_DISABLE_REBUILD')
_COLLECTING_COVERAGE = False
CI = get_boolean_env("CI")
GITHUB_CI = get_boolean_env("GITHUB_CI")
WIN32 = sys.platform == "win32"
BUILD_NATIVE_IMAGE_WITH_ASSERTIONS = get_boolean_env('BUILD_WITH_ASSERTIONS', CI)
GRAALPY_WITH_BOUNCYCASTLE = get_boolean_env("GRAALPY_WITH_BOUNCYCASTLE", False)
mx_gate.add_jacoco_excludes([
"com.oracle.graal.python.pegparser.sst",
"com.oracle.graal.python.pegparser.test",
"com.oracle.truffle.api.staticobject.test",
"com.oracle.truffle.regex.tregex.test",
"com.oracle.truffle.tck",
"com.oracle.truffle.tools.chromeinspector.test",
"com.oracle.truffle.tools.coverage.test",
"com.oracle.truffle.tools.dap.test",
"com.oracle.truffle.tools.profiler.test",
"org.graalvm.tools.insight.test",
"org.graalvm.tools.lsp.test",
])
def is_collecting_coverage():
return bool(mx_gate.get_jacoco_agent_args() or _COLLECTING_COVERAGE)
def wants_debug_build(flags=os.environ.get("CFLAGS", "")):
return any(x in flags for x in ["-g", "-ggdb", "-ggdb3"])
if wants_debug_build():
setattr(mx_native.DefaultNativeProject, "_original_cflags", mx_native.DefaultNativeProject.cflags)
setattr(mx_native.DefaultNativeProject, "cflags", property(
lambda self: self._original_cflags + (["/Z7"] if WIN32 else ["-fPIC", "-ggdb3"])
))
def _libc():
return mx_subst.path_substitutions.substitute("")
def _is_graalos_build():
return "musl" in _libc()
def _with_bouncycastle():
return GRAALPY_WITH_BOUNCYCASTLE and not _is_graalos_build()
def bcflags():
if _with_bouncycastle():
return '--vm.-add-modules=graalpython.bouncycastle,org.bouncycastle.provider,org.bouncycastle.pkix,org.bouncycastle.util'
return ''
if WIN32:
# let's check if VS compilers are on the PATH
if not os.environ.get("LIB"):
mx.log("LIB not in environment, not a VS shell")
elif not os.environ.get("INCLUDE"):
mx.log("INCLUDE not in environment, not a VS shell")
else:
for p in os.environ.get("PATH", "").split(os.pathsep):
if os.path.isfile(os.path.join(os.path.abspath(p), "cl.exe")):
mx.log("LIB and INCLUDE set, cl.exe on PATH, assuming this is a VS shell")
os.environ["DISTUTILS_USE_SDK"] = "1"
if not os.environ.get("MSSdk"):
os.environ["MSSdk"] = os.environ.get("WindowsSdkDir", "unset")
break
else:
mx.log("cl.exe not on PATH, not a VS shell")
def _get_stdlib_home():
return os.path.join(SUITE.dir, "graalpython", "lib-python", "3")
def _get_capi_home():
native_libs_output = mx.distribution("GRAALPYTHON_NATIVE_LIBS").get_output()
assert native_libs_output
return os.path.join(native_libs_output, mx.get_os(), mx.get_arch())
def _extract_graalpython_internal_options(args):
non_internal = []
additional_dists = []
for arg in args:
# Class path extensions
if arg.startswith('-add-dist='):
additional_dists += [arg[10:]]
else:
non_internal += [arg]
return non_internal, additional_dists
def mx_register_dynamic_suite_constituents(register_project, register_distribution):
if register_project and register_distribution:
isolate_build_options = [
'-H:+DetectUserDirectoriesInImageHeap',
]
meta_pom = None
for dist in SUITE.dists:
if dist.name == 'PYTHON_POM':
meta_pom = dist
assert meta_pom, "Cannot find python meta-POM distribution in the graalpython suite"
mx_truffle.register_polyglot_isolate_distributions(SUITE, register_project, register_distribution, 'python',
'graalpython', meta_pom.name, meta_pom.maven_group_id(), meta_pom.theLicense,
isolate_build_options)
def extend_os_env(**kwargs):
env = os.environ.copy()
env.update(**kwargs)
return env
def delete_bad_env_keys(env):
for k in ["SSL_CERT_FILE", "REQUESTS_CA_BUNDLE"]:
if k in env:
del env[k]
def check_vm(vm_warning=True, must_be_jvmci=False):
if not SUITE_COMPILER:
if must_be_jvmci:
mx.abort('** Error ** : graal compiler was not found!')
if vm_warning:
mx.log('** warning ** : graal compiler was not found!! Executing using standard VM..')
def get_jdk():
return mx.get_jdk()
# Called from suite.py
def graalpy_standalone_deps():
include_truffle_runtime = not mx.env_var_to_bool("EXCLUDE_TRUFFLE_RUNTIME")
deps = mx_truffle.resolve_truffle_dist_names(use_optimized_runtime=include_truffle_runtime)
if _with_bouncycastle():
mx.logv("Including bouncycastle with GraalPy standalone")
deps += [
"graalpython:GRAALPYTHON_BOUNCYCASTLE",
"graalpython:BOUNCYCASTLE-PROVIDER",
"graalpython:BOUNCYCASTLE-PKIX",
"graalpython:BOUNCYCASTLE-UTIL",
]
return deps
def _is_overridden_native_image_arg(prefix):
extras = mx.get_opts().extra_image_builder_argument
return any(arg.startswith(prefix) for arg in extras)
def _normalize_branch_name(branch):
if not branch:
return ""
branch = branch.strip()
for prefix in ("refs/heads/", "origin/"):
if branch.startswith(prefix):
return branch[len(prefix):]
return branch
def _graalpython_commit_is_ancestor_of_branch(commit, branch):
"""Return True if the GraalPy commit is already reachable from the branch."""
# Check the remote-tracking branch, not the local branch or HEAD. Regular branch-run
# workspaces may put a synthetic merge commit on a local branch, while origin/
# still represents the target branch state used to decide whether the GraalPy commit has
# really landed.
return _graalpython_merge_base_with_branch(commit, branch) == commit
def _graalpython_merge_base_with_branch(commit, branch):
if not commit or not branch or not SUITE.vc:
return None
merge_base = SUITE.vc.git_command(
SUITE.dir,
["merge-base", commit, f"origin/{branch}"],
abortOnError=False,
)
return merge_base.strip() if merge_base else None
def _graalpython_target_branch_profile_candidates(commit, branch, max_age_days):
"""Return target-branch ancestor commits that may have CI-generated PGO profiles."""
merge_base = _graalpython_merge_base_with_branch(commit, branch)
if not merge_base:
return []
commit_ts = SUITE.vc.git_command(
SUITE.dir,
["show", "-s", "--format=%ct", merge_base],
abortOnError=False,
)
if not commit_ts:
return []
try:
since_ts = int(commit_ts.strip()) - max_age_days * 24 * 60 * 60
except ValueError:
mx.warn(f"Cannot parse commit timestamp for GraalPy PGO profile search: {commit_ts!r}")
return []
candidates = SUITE.vc.git_command(
SUITE.dir,
["rev-list", "--first-parent", f"--since=@{since_ts}", merge_base],
abortOnError=False,
)
return candidates.splitlines() if candidates else []
def _graalpython_pgo_profile_exists(path):
try:
return os.path.isfile(path or "") and os.path.getsize(path) > 0
except OSError:
return False
def _graalpython_target_import_commit(target_branch):
"""Return the GraalPy commit imported by the target branch's VM suite.
Product PGO profiles are produced by the GraalPy post-merge profile job and
keyed by GraalPy commit. In linked cross-repo PR gates, CI can auto-bump the
VM suite's GraalPy import to the GraalPython PR merge commit, which normally
has no post-merge product profile. For feature branches, product-ee therefore
uses the GraalPy import from the target VM branch instead, bypassing that
CI-generated import bump.
Prefer origin/. Some CI workspaces only have the PR merge commit; in
that case HEAD^1 is the target-side parent and still contains the target
branch import. Local workspaces can fall back to the checked-out VM suite.
The suite file is a literal dict, so parse it instead of executing suite.py.
"""
vm_suite = mx.suite("vm", fatalIfMissing=False)
if not vm_suite or not vm_suite.vc:
mx.warn("Cannot resolve target-branch GraalPy import: mx suite 'vm' is not available")
return None
target_branch = _normalize_branch_name(target_branch)
suite_path = "vm/mx.vm/suite.py"
# Some CI checkouts do not keep origin/. In PR merge checkouts, the
# first parent is the target branch side of the merge and therefore still
# gives us the target branch's imported GraalPy commit without fetching.
suite_text = None
refs = [f"origin/{target_branch}"]
parents = vm_suite.vc.git_command(
vm_suite.vc_dir,
["rev-list", "--parents", "-n", "1", "HEAD"],
abortOnError=False,
)
if parents and len(parents.split()) > 2:
refs.append("HEAD^1")
for ref in refs:
if ref:
suite_text = vm_suite.vc.git_command(
vm_suite.vc_dir,
["show", f"{ref}:{suite_path}"],
abortOnError=False,
)
if suite_text:
break
if not suite_text:
suite_file = os.path.join(vm_suite.dir, f"mx.{vm_suite.name}", "suite.py")
try:
with open(suite_file, encoding="utf-8") as f:
suite_text = f.read()
except OSError:
mx.warn(f"Cannot read {suite_path} to resolve the GraalPy profile source")
return None
try:
suite_node = next(
node.value
for node in ast.parse(suite_text, filename=suite_path).body
if isinstance(node, ast.Assign)
and any(isinstance(target, ast.Name) and target.id == "suite" for target in node.targets)
)
suite = ast.literal_eval(suite_node)
imports = suite.get("imports", {}).get("suites", [])
matches = []
for imported_suite in imports:
version = imported_suite.get("version")
if (
imported_suite.get("name") == "graalpython"
and isinstance(version, str)
and re.match(r"^[0-9a-f]{40}$", version)
):
matches.append(version)
except (StopIteration, SyntaxError, ValueError, TypeError) as e:
mx.warn(f"Cannot evaluate {suite_path} to resolve the GraalPy profile source: {e}")
return None
if len(matches) == 1:
return matches[0]
mx.warn(f"Expected exactly one graalpython import in target vm suite, found {len(matches)}")
return None
def github_ci_build_args():
# Determine memory and parallelism for GitHub CI builds
# Use 90% of available memory up to 14GB, but at least 8GB
# Set cores to number of CPUs if at least 4 cores and enough memory, otherwise 1
total_mem = 0
try:
if mx.is_windows():
for m in subprocess.check_output(['wmic', 'memorychip', 'get', 'capacity'], encoding='utf-8').splitlines():
try:
total_mem += int(m) / 1024**3
except ValueError:
pass
else:
total_mem = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') / 1024**3
except:
total_mem = 16.0
min_bound = 8
max_mem = 14*1024
min_mem = int(1024 * (total_mem if total_mem < min_bound else total_mem * .9))
os_cpu = os.cpu_count() or int(os.environ.get("NUMBER_OF_PROCESSORS", 1)) or 1
build_mem = min(min_mem, max_mem)
parallelism = os_cpu if os_cpu >= 4 and build_mem >= min_bound*1024 else 1
return ["-Ob",
f"-J-Xms{build_mem}m",
f"--parallelism={parallelism}"
]
def libpythonvm_build_args():
build_args = []
if os.environ.get("GITHUB_CI"):
build_args += github_ci_build_args()
if graalos := _is_graalos_build():
build_args += ['-H:+GraalOS']
else:
build_args += [
"-Dpolyglot.image-build-time.PreinitializeContexts=python",
"-H:+UnlockExperimentalVMOptions",
'-H:+RelativeCodePointers',
"-H:-UnlockExperimentalVMOptions",
]
if (
not graalos
and mx_sdk_vm_ng.is_nativeimage_ee()
and not os.environ.get('NATIVE_IMAGE_AUXILIARY_ENGINE_CACHE')
and not _is_overridden_native_image_arg("--gc")
):
build_args += ['-H:-ProtectionKeys']
profile = None
require_profile = get_boolean_env("GRAALPY_REQUIRE_PGO_PROFILE")
if (
"GRAALPY_PGO_PROFILE" not in os.environ
and mx.suite('graalpython-enterprise', fatalIfMissing=False)
and mx_sdk_vm_ng.get_bootstrap_graalvm_version() >= mx.VersionSpec("25.0")
and not _is_overridden_native_image_arg("--pgo")
):
vc = SUITE.vc
source_commit = str(vc.tip(SUITE.dir)).strip()
source_branch = _normalize_branch_name(
os.environ.get("FROM_BRANCH") or vc.active_branch(SUITE.dir, abortOnError=False) or 'master'
)
target_branch = _normalize_branch_name(os.environ.get("TO_BRANCH"))
profile_source_commit = source_commit
profile_source_branch = source_branch
profile_source_reason = "current GraalPy commit"
profile_source_candidates = [(profile_source_commit, profile_source_branch, profile_source_reason)]
override = os.environ.get("GRAALPY_PGO_PROFILE_SOURCE_COMMIT")
if override:
profile_source_commit = override.strip()
if not re.match(r"^[0-9a-f]{40}$", profile_source_commit):
mx.abort(f"GRAALPY_PGO_PROFILE_SOURCE_COMMIT must be a 40-character lowercase git commit, got: {override}")
profile_source_reason = "GRAALPY_PGO_PROFILE_SOURCE_COMMIT"
profile_source_candidates = [(profile_source_commit, profile_source_branch, profile_source_reason)]
elif (
target_branch
and source_branch
and source_branch != target_branch
and not (source_branch == MAIN_BRANCH or source_branch.startswith(("release/", "cpu/")))
and not _graalpython_commit_is_ancestor_of_branch(source_commit, target_branch)
):
# Feature branch commits usually have no released-product profile yet,
# but use one if it was explicitly generated. Otherwise prefer the
# nearest available target-branch ancestor profile, then fall back to
# the VM suite's target import for compatibility with older CI workspaces.
profile_search_days = 14
target_candidates = _graalpython_target_branch_profile_candidates(
source_commit, target_branch, profile_search_days
)
target_import_commit = _graalpython_target_import_commit(target_branch)
profile_source_candidates = [
(source_commit, source_branch, "current GraalPy branch commit"),
] + [
(commit, target_branch, f"target branch ancestor from {target_branch}")
for commit in target_candidates
]
if (
target_import_commit
and target_import_commit not in {commit for commit, _, _ in profile_source_candidates}
):
profile_source_candidates.append(
(target_import_commit, target_branch, f"target branch import from {target_branch}")
)
artifact_name = f"{GRAALPY_PGO_PROFILE_ARTIFACT_GROUP}/{GRAALPY_PGO_PROFILE_ARTIFACT_PREFIX}{profile_source_commit}"
mx.log(f"GraalPy source commit for PGO profile lookup: {source_commit}")
if script := os.environ.get("ARTIFACT_DOWNLOAD_SCRIPT"):
# This is always available in the GraalPy CI
profile = f"cached_profile.iprof.gz"
for candidate_commit, candidate_branch, candidate_reason in profile_source_candidates:
profile_source_commit = candidate_commit
profile_source_branch = candidate_branch
profile_source_reason = candidate_reason
artifact_name = f"{GRAALPY_PGO_PROFILE_ARTIFACT_GROUP}/{GRAALPY_PGO_PROFILE_ARTIFACT_PREFIX}{profile_source_commit}"
mx.log(f"GraalPy PGO profile source commit: {profile_source_commit} ({profile_source_reason})")
if os.path.exists(profile):
os.remove(profile)
run(
[
sys.executable,
script,
artifact_name,
profile,
],
nonZeroIsFatal=False,
)
if _graalpython_pgo_profile_exists(profile):
break
elif not require_profile:
mx.log(f"GraalPy PGO profile source commit: {profile_source_commit} ({profile_source_reason})")
# Locally, we try to get a reasonable profile
get_profile = mx.command_function('python-get-latest-profile', fatalIfMissing=False)
if get_profile:
seen_branches = set()
for b in [profile_source_branch, source_branch, MAIN_BRANCH]:
b = _normalize_branch_name(b)
if not b or b in seen_branches:
continue
seen_branches.add(b)
if not profile:
try:
profile = get_profile(["--branch", b])
except BaseException:
pass
profile_missing = not _graalpython_pgo_profile_exists(profile)
if require_profile and profile_missing:
mx.abort("\n".join([
"GRAALPY_REQUIRE_PGO_PROFILE is set, but no CI generated GraalPy PGO profile was found.",
f"GraalPy source commit: {source_commit}",
f"Source branch: {source_branch or ''}",
f"Target branch: {target_branch or ''}",
f"PGO profile source commit: {profile_source_commit} ({profile_source_reason})",
f"Expected artifact: {artifact_name}",
"The product profile configuration does not fall back to benchmark-local PGO.",
"Run the GraalPy CI job python-pgo-profile-post_merge-linux-amd64-jdk-latest for the PGO profile source commit, then retry the product-ee benchmark.",
]))
if CI and profile_missing:
mx.log("No profile in CI job")
# When running on a release branch or attempting to merge into
# a release/CPU branch, make sure we can use a PGO profile, and
# when running in the CI on a bench runner, ensure a PGO profile.
if (
any(
_normalize_branch_name(b).startswith(("release/", "cpu/"))
for b in [source_branch, target_branch]
)
or ("bench" in os.environ.get('BUILD_NAME', ''))
):
mx.warn("PGO profile must exist for benchmarking and release, creating one now...")
profile = graalpy_native_pgo_build_and_test()
if _graalpython_pgo_profile_exists(profile):
print(invert(f"Automatically chose PGO profile {profile}. To disable this, set GRAALPY_PGO_PROFILE to an empty string'", blinking=True), file=sys.stderr)
build_args += [
f"--pgo={profile}",
"-H:+UnlockExperimentalVMOptions",
"-H:+PGOPrintProfileQuality",
"-H:-UnlockExperimentalVMOptions",
]
else:
print(invert("Not using an automatically selected PGO profile"), file=sys.stderr)
return build_args
def graalpy_native_pgo_build_and_test(args=None):
"""
Builds a PGO-instrumented GraalPy native standalone, runs the unittests to generate a profile,
then builds a PGO-optimized GraalPy native standalone with the collected profile.
The profile file will be named 'default.iprof' in native image build directory.
"""
if mx_sdk_vm_ng.get_bootstrap_graalvm_version() < mx.VersionSpec("25.0"):
mx.abort("python-native-pgo not supported on GraalVM < 25")
host_inlining_log = Path(SUITE.dir) / "host-inlining.txt"
host_inlining_log_gz = Path(str(host_inlining_log) + ".gz")
if host_inlining_log.exists():
host_inlining_log.unlink()
if host_inlining_log_gz.exists():
host_inlining_log_gz.unlink()
with set_env(GRAALPY_PGO_PROFILE=""):
mx.log(mx.colorize("[PGO] Building PGO-instrumented native image", color="yellow"))
build_home = graalpy_standalone_home('native', enterprise=True, build=True)
instrumented_home = build_home + "_PGO_INSTRUMENTED"
shutil.rmtree(instrumented_home, ignore_errors=True)
shutil.copytree(build_home, instrumented_home, symlinks=True, ignore_dangling_symlinks=True)
instrumented_launcher = os.path.join(instrumented_home, 'bin', _graalpy_launcher())
mx.log(mx.colorize(f"[PGO] Instrumented build complete: {instrumented_home}", color="yellow"))
mx.log(mx.colorize(f"[PGO] Running graalpytest with instrumented binary: {instrumented_launcher}", color="yellow"))
with tempfile.TemporaryDirectory() as d:
with set_env(
GRAALPYTEST_ALLOW_NO_JAVA_ASSERTIONS="true",
GRAAL_PYTHON_VM_ARGS="\v".join([
f"--vm.XX:ProfilesDumpFile={os.path.join(d, '$UUID$.iprof')}",
f"--vm.XX:ProfilesLCOVFile={os.path.join(d, '$UUID$.info')}",
]),
GRAALPY_HOME=instrumented_home,
):
graalpytest(["--python", instrumented_launcher, "test_venv.py"])
mx.command_function('benchmark')(["meso-small:*", "--", "--python-vm", "graalpython", "--python-vm-config", 'custom'])
iprof_path = Path(SUITE.dir) / f'default.iprof'
lcov_path = Path(SUITE.dir) / f'default.lcov'
run([
os.path.join(
graalvm_jdk(enterprise=True),
"bin",
f"native-image-configure{'.exe' if mx.is_windows() else ''}",
),
"merge-pgo-profiles",
f"--input-dir={d}",
f"--output-file={iprof_path}"
])
run([
"/usr/bin/env",
"lcov",
"-o", str(lcov_path),
*itertools.chain.from_iterable([
["-a", f.absolute().as_posix()] for f in Path(d).glob("*.info")
])
], nonZeroIsFatal=False)
run([
"/usr/bin/env",
"genhtml",
"--source-directory", str(Path(SUITE.dir) / "com.oracle.graal.python" / "src"),
"--source-directory", str(Path(SUITE.dir) / "com.oracle.graal.python.pegparser" / "src"),
"--source-directory", str(Path(SUITE.get_output_root()) / "com.oracle.graal.python" / "src_gen"),
"--include", "com/oracle/graal/python",
"--keep-going",
"-o", "lcov_html",
str(lcov_path),
], nonZeroIsFatal=False)
if not os.path.isfile(iprof_path):
mx.abort(f"[PGO] Could not find profile file at expected location: {iprof_path}")
with set_env(GRAALPY_PGO_PROFILE=str(iprof_path), GRAALPY_HOST_INLINING_LOG=str(host_inlining_log)):
mx.log(mx.colorize("[PGO] Building optimized native image with collected profile", color="yellow"))
native_bin = graalpy_standalone('native', enterprise=True, build=True)
mx.log(mx.colorize(f"[PGO] Optimized PGO build complete: {native_bin}", color="yellow"))
if host_inlining_log.exists():
mx.log(mx.colorize(f"[PGO] Host inlining log at: {host_inlining_log}", color="yellow"))
with open(host_inlining_log, 'rb') as f_in, gzip.open(host_inlining_log_gz, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
host_inlining_log.unlink()
mx.log(mx.colorize(f"[PGO] Gzipped host inlining log at: {host_inlining_log_gz}", color="yellow"))
else:
mx.warn(f"[PGO] Host inlining log was not produced at expected location: {host_inlining_log}")
iprof_gz_path = str(iprof_path) + '.gz'
with open(iprof_path, 'rb') as f_in, gzip.open(iprof_gz_path, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
mx.log(mx.colorize(f"[PGO] Gzipped profile at: {iprof_gz_path}", color="yellow"))
if script := os.environ.get("ARTIFACT_UPLOADER_SCRIPT"):
commit = str(SUITE.vc.tip(SUITE.dir)).strip()
run(
[
sys.executable,
script,
iprof_gz_path,
f"{GRAALPY_PGO_PROFILE_ARTIFACT_PREFIX}{commit}",
GRAALPY_PGO_PROFILE_ARTIFACT_GROUP,
"--lifecycle",
"cache",
"--artifact-repo-key",
os.environ.get("ARTIFACT_REPO_KEY_LOCATION"),
'--skip-existing',
],
)
if args is None:
return iprof_gz_path
def full_python(args, env=None):
"""Run python from standalone build (unless kwargs are given). Does not build GraalPython sources automatically."""
if not any(arg.startswith('--python.WithJavaStacktrace') for arg in args):
args.insert(0, '--python.WithJavaStacktrace=1')
if "--hosted" in args[:2]:
return do_run_python(args)
if '--vm.da' not in args:
args.insert(0, '--vm.ea')
if not any(arg.startswith('--experimental-options') for arg in args):
args.insert(0, '--experimental-options')
handle_debug_arg(args)
for arg in itertools.chain(
itertools.chain(*map(shlex.split, reversed(mx._opts.java_args_sfx))),
reversed(shlex.split(mx._opts.java_args if mx._opts.java_args else "")),
itertools.chain(*map(shlex.split, reversed(mx._opts.java_args_pfx))),
):
if arg.startswith("-"):
args.insert(0, f"--vm.{arg[1:]}")
else:
mx.warn(f"Dropping {arg}, cannot pass it to launcher")
standalone_home = graalpy_standalone_home('jvm', dev=True, build=False)
graalpy_path = os.path.join(standalone_home, 'bin', _graalpy_launcher())
if not os.path.exists(graalpy_path):
mx.abort("GraalPy standalone doesn't seem to be built.\n" +
"To build it: mx python-jvm")
run([graalpy_path] + args, env=env)
def handle_debug_arg(args):
if mx._opts.java_dbg_port:
args.insert(0,
f"--vm.agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=127.0.0.1:{mx._opts.java_dbg_port}")
def do_run_python(args, extra_vm_args=None, env=None, jdk=None, extra_dists=None, cp_prefix=None, cp_suffix=None, main_class=GRAALPYTHON_MAIN_CLASS, minimal=False, **kwargs):
if "--hosted" in args[:2]:
args.remove("--hosted")
if not any(arg.startswith('--python.WithJavaStacktrace') for arg in args):
args.insert(0, '--python.WithJavaStacktrace=1')
if not any(arg.startswith("--python.CAPI") for arg in args):
capi_home = _get_capi_home()
args.insert(0, "--python.CAPI=%s" % capi_home)
args.insert(0, "--experimental-options")
if not env:
env = os.environ.copy()
env.setdefault("GRAAL_PYTHONHOME", _dev_pythonhome())
delete_bad_env_keys(env)
check_vm_env = env.get('GRAALPYTHON_MUST_USE_GRAAL', False)
if check_vm_env:
if check_vm_env == '1':
check_vm(must_be_jvmci=True)
elif check_vm_env == '0':
check_vm()
if minimal:
x = [x for x in SUITE.dists if x.name == "GRAALPYTHON"][0]
dists = [dep for dep in x.deps if dep.isJavaProject() or dep.isJARDistribution() and dep.exists()]
# Hack: what we should just do is + ['GRAALPYTHON_VERSIONS_MAIN'] and let MX figure out
# the class-path and other VM arguments necessary for it. However, due to a bug in MX,
# LayoutDirDistribution causes an exception if passed to mx.get_runtime_jvm_args,
# because it does not properly initialize its super class ClasspathDependency, see MX PR: 1665.
ver_dep = mx.distribution('GRAALPYTHON_VERSIONS_MAIN').get_output()
cp_prefix = ver_dep if cp_prefix is None else (str(ver_dep) + os.pathsep + cp_prefix)
else:
dists = ['GRAALPYTHON']
dists += ['GRAALPYTHON-LAUNCHER']
vm_args, graalpython_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=False)
if minimal:
vm_args.insert(0, f"-Dorg.graalvm.language.python.home={_dev_pythonhome()}")
graalpython_args, additional_dists = _extract_graalpython_internal_options(graalpython_args)
dists += additional_dists
if extra_dists:
dists += extra_dists
if not CI:
# Try eagerly to include tools for convenience when running Python
if not mx.suite("tools", fatalIfMissing=False):
SUITE.import_suite("tools", version=None, urlinfos=None, in_subdir=True)
if mx.suite("tools", fatalIfMissing=False):
for tool in ["CHROMEINSPECTOR", "TRUFFLE_COVERAGE"]:
if os.path.exists(mx.distribution(tool).path):
dists.append(tool)
graalpython_args.insert(0, '--experimental-options=true')
vm_args += mx.get_runtime_jvm_args(dists, jdk=jdk, cp_prefix=cp_prefix, cp_suffix=cp_suffix, force_cp=True)
if not jdk:
jdk = get_jdk()
# default: assertion checking is enabled
if extra_vm_args is None or '-da' not in extra_vm_args:
vm_args += ['-ea', '-esa']
if extra_vm_args:
vm_args += extra_vm_args
vm_args.append(main_class)
return mx.run_java(vm_args + graalpython_args, jdk=jdk, env=env, **kwargs)
def _dev_pythonhome_context():
home = os.environ.get("GRAAL_PYTHONHOME", _dev_pythonhome())
return set_env(GRAAL_PYTHONHOME=home)
def _dev_pythonhome():
return os.path.join(SUITE.dir, "graalpython")
DELVEEWHEEL_GRAALPY_ARTIFACT = "graal/python-native-standalone-svm-svmee-java25-windows-amd64-25.2.4.zip"
DELVEEWHEEL_GRAALPY_HOME = "graalpy3.12-25.2.4-windows-amd64"
def _downloaded_graalpy_for_delvewheel():
download_script = os.environ.get("ARTIFACT_DOWNLOAD_SCRIPT")
if not download_script:
mx.abort("Cannot build delvewheel venv: need CPython >= 3.12 or ARTIFACT_DOWNLOAD_SCRIPT")
cache_dir = Path(SUITE.get_output_root()).absolute() / "delvewheel-graalpy"
archive = cache_dir / Path(DELVEEWHEEL_GRAALPY_ARTIFACT).name
extracted = cache_dir / "extracted"
graalpy = extracted / DELVEEWHEEL_GRAALPY_HOME / "bin" / "graalpy.exe"
if graalpy.exists():
return str(graalpy)
cache_dir.mkdir(parents=True, exist_ok=True)
if not archive.exists():
mx.log(
f"{time.strftime('[%H:%M:%S] ')} Downloading GraalPy for delvewheel venv: "
f"{DELVEEWHEEL_GRAALPY_ARTIFACT}"
)
subprocess.check_call([sys.executable, download_script, DELVEEWHEEL_GRAALPY_ARTIFACT, str(archive)])
extracted.mkdir(parents=True, exist_ok=True)
mx.Extractor.create(str(archive)).extract(str(extracted))
if not graalpy.exists():
mx.abort(f"Could not find bin/graalpy.exe in downloaded artifact {archive}")
return str(graalpy)
def get_path_with_patchelf():
path = os.environ.get("PATH", "")
if mx.is_linux() and not shutil.which("patchelf"):
venv = Path(SUITE.get_output_root()).absolute() / "patchelf-venv"
path += os.pathsep + str(venv / "bin")
if not shutil.which("patchelf", path=path):
mx.log(f"{time.strftime('[%H:%M:%S] ')} Building patchelf-venv with {sys.executable}... [patchelf not found on PATH]")
t0 = time.time()
subprocess.check_call([sys.executable, "-m", "venv", str(venv)])
subprocess.check_call([str(venv / "bin" / "pip"), "install", "patchelf"])
mx.log(f"{time.strftime('[%H:%M:%S] ')} Building patchelf-venv with {sys.executable}... [duration: {time.time() - t0}]")
if mx.is_windows() and HAS_JEP_454 and not shutil.which("delvewheel"):
venv = Path(SUITE.get_output_root()).absolute() / "delvewheel-venv"
path += os.pathsep + str(venv / "Scripts")
if not shutil.which("delvewheel", path=path):
if sys.implementation.name == "cpython" and sys.version_info >= (3, 12):
venv_python = [sys.executable]
else:
venv_python = [_downloaded_graalpy_for_delvewheel(), "-X", "jit=0"]
mx.log(
f"{time.strftime('[%H:%M:%S] ')} Building delvewheel-venv with {shlex.join(venv_python)}... "
"[delvewheel not found on PATH]"
)
t0 = time.time()
subprocess.check_call(venv_python + ["-m", "venv", str(venv)])
subprocess.check_call(
[
str(venv / "Scripts" / "python.exe"),
"-m",
"pip",
"install",
"delvewheel>=1.13.0",
]
)
mx.log(
f"{time.strftime('[%H:%M:%S] ')} Building delvewheel-venv with {shlex.join(venv_python)}... "
f"[duration: {time.time() - t0}]"
)
return path
def punittest(ars, report: Union[Task, bool, None] = False):
"""
Runs GraalPython junit tests and memory leak tests, which can be skipped using --no-leak-tests.
Pass --regex to further filter the junit and TSK tests. GraalPy tests are always run in two configurations:
with language home on filesystem and with language home served from the Truffle resources.
"""
path = get_path_with_patchelf()
args = [] if ars is None else ars
@dataclass
class TestConfig:
identifier: str
args: list
useResources: bool
reportConfig: Union[Task, bool, None] = report
def __str__(self):
return f"args={self.args!r}, useResources={self.useResources}, report={self.reportConfig}"
def __post_init__(self):
assert ' ' not in self.identifier
configs = []
skip_leak_tests = False
if "--no-leak-tests" in args:
skip_leak_tests = True
args.remove("--no-leak-tests")
if is_collecting_coverage():
skip_leak_tests = True
vm_args = ['-Dpolyglot.engine.WarnInterpreterOnly=false']
if mx.suite('compiler', fatalIfMissing=False):
vm_args.append('-Dpolyglot.engine.CompilationFailureAction=ExitVM')
# Note: we must use filters instead of --regex so that mx correctly processes the unit test configs,
# but it is OK to apply --regex on top of the filters
graalpy_tests = ['com.oracle.graal.python.test', 'com.oracle.graal.python.pegparser.test', 'org.graalvm.python.embedding.test']
configs += [
TestConfig("junit", vm_args + graalpy_tests + args, True),
TestConfig("junit", vm_args + graalpy_tests + args, False)]
if not mx.is_windows():
configs += [
# Tests that must run in their own process due to C extensions usage, for now ignored on Windows
TestConfig("multi-threaded-import-java", vm_args + ['com.oracle.graal.python.cext.test.MultithreadedImportTestNative'] + args, True),
TestConfig("multi-threaded-import-java", vm_args + ['com.oracle.graal.python.cext.test.MultithreadedImportTestNative'] + args, False),
TestConfig("multi-threaded-import-native", vm_args + ['com.oracle.graal.python.cext.test.MultithreadedImportTestJava'] + args, True),
TestConfig("multi-threaded-import-native", vm_args + ['com.oracle.graal.python.cext.test.MultithreadedImportTestJava'] + args, False),
]
if '--regex' not in args:
async_regex = ['--regex', r'com\.oracle\.graal\.python\.test\.integration\.advanced\.AsyncActionThreadingTest']
configs.append(TestConfig("async", vm_args + ['-Dpython.AutomaticAsyncActions=false', 'com.oracle.graal.python.test', 'org.graalvm.python.embedding.test'] + async_regex + args, True, False))
else:
skip_leak_tests = True
for c in configs:
mx.log(f"Python JUnit tests configuration: {c}")
PythonMxUnittestConfig.useResources = c.useResources
with set_env(PATH=path):
mx_unittest.unittest(c.args, test_report_tags=({"task": f"punittest-{c.identifier}-{'w' if c.useResources else 'wo'}-resources"} if c.reportConfig else None))
if skip_leak_tests:
return
# test leaks with Python code only
run_leak_launcher(["--code", "pass", ])
run_leak_launcher(["--repeat-and-check-size", "250", "--null-stdout", "--code", "print('hello')"])
c_api_leak_test = (
'import _testcapi, _testlimitedcapi; '
't = _testlimitedcapi.tuple_pack(2, "a", "b"); '
'assert _testcapi.tuple_get_item(t, 1) == "b"'
)
# test leaks when some C module code is involved
if HAS_JEP_454:
run_leak_launcher([
"--forbid-capi-residue", "--code",
c_api_leak_test,
])
# test leaks with shared engine Python code only
run_leak_launcher(["--shared-engine", "--code", "pass"])
run_leak_launcher(["--shared-engine", "--repeat-and-check-size", "250", "--null-stdout", "--code", "print('hello')"])
# test leaks with shared engine when some C module code is involved
if HAS_JEP_454:
run_leak_launcher([
"--shared-engine", "--forbid-capi-residue", "--code",
c_api_leak_test,
])
run_leak_launcher(["--shared-engine", "--code", '[10, 20]', "--python.UseNativePrimitiveStorageStrategy=true",
"--forbidden-class", "com.oracle.graal.python.runtime.sequence.storage.NativePrimitiveSequenceStorage",
"--forbidden-class", "com.oracle.graal.python.runtime.native_memory.NativePrimitiveReference"])
run_leak_launcher(["--code", '[10, 20]', "--python.UseNativePrimitiveStorageStrategy=true",
"--forbidden-class", "com.oracle.graal.python.runtime.sequence.storage.NativePrimitiveSequenceStorage",
"--forbidden-class", "com.oracle.graal.python.runtime.native_memory.NativePrimitiveReference"])
def verify_junit_compilation_failure():
compiler_failure_exit_test = 'com.oracle.graal.python.test.advanced.CompilerFailureExitTest'
enable_flag_env_name = 'GRAALPYTHON_JUNIT_COMPILER_FAILURE_EXIT_TEST'
if not mx.suite('compiler', fatalIfMissing=False):
mx.warn(f"Skipping {compiler_failure_exit_test}: the compiler suite is not imported.")
return
args = [
'unittest',
'--suite', 'graalpython',
'--verbose',
'-Dpolyglot.engine.CompilationFailureAction=ExitVM',
] + [
compiler_failure_exit_test,
]
output = mx.OutputCapture()
exit_code = run_mx(args, nonZeroIsFatal=False, out=mx.TeeOutputCapture(output),
err=mx.TeeOutputCapture(output), env={**os.environ, enable_flag_env_name: 'true'})
if exit_code == 0:
mx.abort(f"Expected {compiler_failure_exit_test} to exit the VM with a compiler failure, but it passed.")
if 'Graal compilation failure' not in output.data:
mx.abort(f"{compiler_failure_exit_test} failed, but not with the expected compiler-failure exit.")
PYTHON_ARCHIVES = ["GRAALPYTHON_GRAALVM_SUPPORT"]
PYTHON_NATIVE_PROJECTS = ["python-libbz2",
"python-liblzma",
"python-libzsupport",
"python-libposix",
"com.oracle.graal.python.cext"]
def nativebuild(_):
"Build the non-Java Python projects and archives"
mx.build(["--dependencies", ",".join(PYTHON_NATIVE_PROJECTS + PYTHON_ARCHIVES)])
def nativeclean(_):
"Clean the non-Java Python projects"
mx.clean(["--dependencies", ",".join(PYTHON_NATIVE_PROJECTS + PYTHON_ARCHIVES)])
class GraalPythonTags(object):
junit = 'python-junit'
junit_maven = 'python-junit-maven'
junit_maven_isolates = 'python-junit-polyglot-isolates'
jvmbuild = 'python-jvm-build'
unittest = 'python-unittest'
unittest_bouncycastle = 'python-unittest-bouncycastle'
unittest_cpython = 'python-unittest-cpython'
unittest_sandboxed = 'python-unittest-sandboxed'
unittest_multi = 'python-unittest-multi-context'
unittest_multi_sandboxed = 'python-unittest-multi-context-sandboxed'
unittest_jython = 'python-unittest-jython'
unittest_arrow = 'python-unittest-arrow-storage'
unittest_standalone = 'python-unittest-standalone'
tagged = 'python-tagged-unittest'
svmbuild = 'python-svm-build'
svm_graalos_standalone_build = 'python-svm-graalos-standalone-build'
svmunit = 'python-svm-unittest'
svmunit_sandboxed = 'python-svm-unittest-sandboxed'
graalvm = 'python-graalvm'
embedding = 'python-standalone-embedding'
graalvm_sandboxed = 'python-graalvm-sandboxed'
svm = 'python-svm'
native_image_embedder = 'python-native-image-embedder'
license = 'python-license'
language_checker = 'python-language-checker'
exclusions_checker = 'python-class-exclusion-checker'
def python_gate(args):
if not os.environ.get("JDT"):
os.environ["JDT"] = "builtin"
if not os.environ.get("ECLIPSE_EXE"):
find_eclipse()
if "--tags" not in args:
args += ["--tags"]
tags = ["style"]
for x in dir(GraalPythonTags):
v = getattr(GraalPythonTags, x)
if isinstance(v, str) and v.startswith("python-"):
if "sandboxed" not in v and "bouncycastle" not in v:
tags.append(v)
args.append(",".join(tags))
mx.log("Running mx python-gate " + " ".join(args))
return mx.command_function("gate")(args)
python_gate.__doc__ = 'Custom gates are %s' % ", ".join([
getattr(GraalPythonTags, t) for t in dir(GraalPythonTags) if not t.startswith("__")
])
def find_eclipse():
pardir = os.path.abspath(os.path.join(SUITE.dir, ".."))
for f in [os.path.join(SUITE.dir, f)
for f in os.listdir(SUITE.dir)] + [os.path.join(pardir, f)
for f in os.listdir(pardir)]:
if os.path.basename(f) == "eclipse" and os.path.isdir(f):
mx.log("Automatically choosing %s for Eclipse" % f)
eclipse_exe = os.path.join(f, f"eclipse{'.exe' if mx.is_windows() else ''}")
if os.path.exists(eclipse_exe):
os.environ["ECLIPSE_EXE"] = eclipse_exe
return
@contextlib.contextmanager
def set_env(**environ):
"""Temporarily set the process environment variables"""
old_environ = dict(os.environ)
for k, v in environ.items():
if v is None:
if k in os.environ:
del os.environ[k]
else:
os.environ[k] = v
try:
yield
finally:
os.environ.clear()
os.environ.update(old_environ)
def _graalpy_launcher():
name = 'graalpy'
return f"{name}.exe" if WIN32 else name
# dev only has effect if standalone_type is 'jvm' and means minimal, Default TruffleRuntime (no JIT)
def graalpy_standalone_home(standalone_type, enterprise=False, dev=False, build=True):
assert standalone_type in ['native', 'jvm']
jdk_version = mx.get_jdk().version
# Check if GRAALPY_HOME points to some compatible pre-built GraalPy standalone
python_home = os.environ.get("GRAALPY_HOME", None)
if python_home:
python_home = os.path.abspath(glob.glob(python_home)[0])
mx.logv("Using GraalPy standalone from GRAALPY_HOME: " + python_home)
# Try to verify that we're getting what we expect:
has_java = os.path.exists(os.path.join(python_home, 'jvm', 'bin', mx.exe_suffix('java')))
if has_java != (standalone_type == 'jvm'):
mx.abort(f"GRAALPY_HOME is not compatible with the requested distribution type.\n"
f"jvm/bin/java exists?: {has_java}, requested type={standalone_type}.")
line = ''
with open(os.path.join(python_home, 'release'), 'r') as f:
while 'JAVA_VERSION=' not in line:
line = f.readline()
if 'JAVA_VERSION=' not in line:
mx.abort(f"GRAALPY_HOME does not contain 'release' file. Cannot check Java version.")
actual_jdk_version = mx.VersionSpec(line.strip('JAVA_VERSION=').strip(' "\n\r'))
if actual_jdk_version != jdk_version:
mx.abort(f"GRAALPY_HOME is not compatible with the requested JDK version.\n"
f"actual version: '{actual_jdk_version}', version string: {line}, requested version: {jdk_version}.")
return python_home
# Build
if standalone_type == 'jvm':
if dev or not HAS_JEP_454:
env_file = 'jvm'
else:
env_file = 'jvm-ee-libgraal' if enterprise else 'jvm-ce-libgraal'
standalone_dist = 'GRAALPY_JVM_STANDALONE'
if "GraalVM" in subprocess.check_output([get_jdk().java, '-version'], stderr=subprocess.STDOUT, universal_newlines=True):
env_file = ""
else:
env_file = 'native-ee' if enterprise else 'native-ce'
standalone_dist = 'GRAALPY_NATIVE_STANDALONE'
mx_args = ['-p', SUITE.dir, *(['--env', env_file] if env_file else [])]
if GITHUB_CI:
mx_args.append("--extra-image-builder-argument=-Ob")
else:
mx_args.append("--extra-image-builder-argument=-g")
pgo_profile = os.environ.get("GRAALPY_PGO_PROFILE")
if pgo_profile is not None:
if not enterprise or standalone_type != "native":
mx.abort("PGO is only supported on enterprise NI")
if pgo_profile:
mx_args.append(f"--extra-image-builder-argument=--pgo={pgo_profile}")
mx_args.append(f"--extra-image-builder-argument=-H:+UnlockExperimentalVMOptions")
mx_args.append(f"--extra-image-builder-argument=-H:+PGOPrintProfileQuality")
if host_inlining_log := os.environ.get("GRAALPY_HOST_INLINING_LOG"):
mx_args.extend([
f"--extra-image-builder-argument=-H:Log=HostInliningPhase,~CanonicalizerPhase,~GraphBuilderPhase",
f"--extra-image-builder-argument=-H:+TruffleHostInliningPrintExplored",
f"--extra-image-builder-argument=-H:MethodFilter=com.oracle.graal.python.*.*",
f"--extra-image-builder-argument=-H:-UnlockExperimentalVMOptions",
f"--extra-image-builder-argument=-Dgraal.LogFile={host_inlining_log}",
])
else:
mx_args.append(f"--extra-image-builder-argument=--pgo-instrument")
mx_args.append(f"--extra-image-builder-argument=-H:+UnlockExperimentalVMOptions")
mx_args.append(f"--extra-image-builder-argument=-H:+ProfilingLCOV")
elif BUILD_NATIVE_IMAGE_WITH_ASSERTIONS:
mx_args.append("--extra-image-builder-argument=-ea")
mx_args.append("--extra-image-builder-argument=-J-ea")
if mx_gate.get_jacoco_agent_args() or (build and not DISABLE_REBUILD):
# This build is purposefully done without the LATEST_JAVA_HOME in the
# environment, so we can build JVM standalones on an older Graal JDK
run_mx(mx_args + ["build", "--target", standalone_dist])
python_home = os.path.join(SUITE.dir, 'mxbuild', f"{mx.get_os()}-{mx.get_arch()}", standalone_dist)
if standalone_type == 'native':
debuginfo = os.path.join(SUITE.dir, 'mxbuild', f"{mx.get_os()}-{mx.get_arch()}", "libpythonvm", "libpythonvm.so.debug")
if os.path.exists(debuginfo):
shutil.copy(debuginfo, os.path.join(python_home, 'lib'))
return python_home
def graalpy_standalone(standalone_type, enterprise=False, dev=False, build=True):
assert standalone_type in ['native', 'jvm']
if standalone_type == 'native' and mx_gate.get_jacoco_agent_args():
return graalpy_standalone('jvm', enterprise=enterprise, dev=dev, build=build)
home = graalpy_standalone_home(standalone_type, enterprise=enterprise, dev=dev, build=build)
launcher = os.path.join(home, 'bin', _graalpy_launcher())
return make_coverage_launcher_if_needed(launcher)
def graalpy_standalone_jvm():
return graalpy_standalone('jvm')
def graalpy_standalone_native():
return graalpy_standalone('native')
def graalpy_standalone_jvm_enterprise():
return os.path.join(graalpy_standalone_home('jvm', enterprise=True), 'bin', _graalpy_launcher())
def graalpy_standalone_native_enterprise():
return os.path.join(graalpy_standalone_home('native', enterprise=True), 'bin', _graalpy_launcher())
def graalvm_jdk(enterprise=False):
jdk_version = mx.get_jdk().version
# Check if GRAAL_JDK_HOME points to some compatible pre-built gvm
graal_jdk_home = os.environ.get("GRAAL_JDK_HOME", None)
if graal_jdk_home:
graal_jdk_home = os.path.abspath(glob.glob(graal_jdk_home)[0])
if sys.platform == "darwin":
jdk_home_subdir = os.path.join(graal_jdk_home, 'Contents', 'Home')
if os.path.exists(jdk_home_subdir):
graal_jdk_home = jdk_home_subdir
mx.logv("Using Graal from GRAAL_JDK_HOME: " + graal_jdk_home)
# Try to verify that we're getting what we expect:
has_java = os.path.exists(os.path.join(graal_jdk_home, 'bin', mx.exe_suffix('java')))
if not has_java:
mx.abort(f"GRAAL_JDK_HOME does not contain java executable.")
release = os.path.join(graal_jdk_home, 'release')
if not os.path.exists(release):
mx.abort(f"No 'release' file in GRAAL_JDK_HOME.")
java_version = None
with open(release, 'r') as f:
while not java_version:
line = f.readline()
if 'JAVA_VERSION=' in line:
java_version = line
if not java_version:
mx.abort(f"Could not check Java version in GRAAL_JDK_HOME 'release' file.")
actual_jdk_version = mx.VersionSpec(java_version.strip('JAVA_VERSION=').strip(' "\n\r'))
if actual_jdk_version.parts[0] != jdk_version.parts[0]:
mx.abort(f"GRAAL_JDK_HOME is not compatible with the requested JDK version.\n"
f"actual version: '{actual_jdk_version}', version string: {java_version}, requested version: {jdk_version}.")
return graal_jdk_home
jdk_major_version = mx.get_jdk().version.parts[0]
if enterprise:
mx_args = ['-p', str(next(Path(mx.suite('truffle').dir).resolve().parent.parent.glob('*/vm-enterprise'))), '--env', 'ee']
edition = ""
else:
mx_args = ['-p', os.path.join(mx.suite('truffle').dir, '..', 'vm'), '--env', 'ce']
edition = "COMMUNITY_"
if not DISABLE_REBUILD:
run_mx(mx_args + ["build", "--dep", f"GRAALVM_{edition}JAVA{jdk_major_version}"], env={**os.environ, **LATEST_JAVA_HOME})
out = mx.OutputCapture()
run_mx(["--quiet"] + mx_args + ["graalvm-home"], out=out)
return out.data.splitlines()[-1].strip()
def get_maven_cache():
buildnr = os.environ.get('BUILD_NUMBER')
# don't worry about maven.repo.local if not running on gate
return os.path.join(SUITE.get_mx_output_dir(), 'm2_cache_' + buildnr) if buildnr else None
def update_maven_opts(env):
m2_cache = get_maven_cache()
if m2_cache:
mvn_repo_local = f'-Dmaven.repo.local={m2_cache}'
maven_opts = env.get('MAVEN_OPTS')
maven_opts = maven_opts + " " + mvn_repo_local if maven_opts else mvn_repo_local
if mx.is_windows():
maven_opts = maven_opts.replace("|", "^|")
env['MAVEN_OPTS'] = maven_opts
mx.log(f"Added '{mvn_repo_local}' to MAVEN_OPTS={maven_opts}")
return env
@contextlib.contextmanager
def use_local_maven_repo_settings(repo_path):
"""Yield Maven user settings that resolve artifacts from ``repo_path`` first.
Maven repositories declared in settings profiles take precedence over repositories from the
project POM. Generate a self-contained user settings file with the local repository followed by
Maven Central URLs processed by mx URL rewriting. In CI, these URLs resolve to the corporate
ArtifactHub mirrors. Maven still merges this user settings file with its global settings.
"""
profile_id = 'graalpy-local-maven-repository'
def repository_xml(repository_id, repository_url, element_name):
return dedent(f"""\
{escape(repository_id)}
{escape(repository_url)}
true
true
""")
repositories: list[tuple[str, str]] = [
('graalpy-local', pathlib.Path(repo_path).resolve().as_uri()),
('graalpy-central', mx_urlrewrites.rewriteurl('https://repo1.maven.org/maven2/')),
]
search_maven_url = 'https://search.maven.org/remotecontent?filepath='
rewritten_search_maven_url = mx_urlrewrites.rewriteurl(search_maven_url)
if rewritten_search_maven_url != search_maven_url:
repositories.append(('graalpy-central-fallback', rewritten_search_maven_url))
repositories_xml = '\n'.join(
indent(repository_xml(*repository, 'repository'), ' ') for repository in repositories
)
plugin_repositories_xml = '\n'.join(
indent(repository_xml(*repository, 'pluginRepository'), ' ') for repository in repositories
)
settings_xml = f"""
{profile_id}
{repositories_xml}
{plugin_repositories_xml}
{profile_id}
"""
with tempfile.NamedTemporaryFile(mode='w', suffix='.xml', encoding='UTF-8', delete=False) as settings:
settings.write(settings_xml)
settings_path = settings.name
try:
yield settings_path
finally:
os.unlink(settings_path)
def deploy_library_to_local_maven_repo(library_name, repo_url, env):
library = mx.library(library_name)
if not hasattr(library, 'maven'):
mx.abort(f'Cannot deploy {library_name}: library does not define Maven metadata')
maven = library.maven
mx.run_maven([
'deploy:deploy-file',
'-DrepositoryId=local',
f'-Durl={repo_url}',
f'-DgroupId={maven["groupId"]}',
f'-DartifactId={maven["artifactId"]}',
f'-Dversion={maven["version"]}',
f'-Dfile={library.get_path(True)}',
'-Dpackaging=jar',
'-DgeneratePom=true',
'-DretryFailedDeploymentCount=10',
], env=env)
def deploy_local_maven_repo(env=None):
env = update_maven_opts({**os.environ.copy(), **(env or {})})
run_mx_args = [
'-p',
os.path.join(mx.suite('truffle').dir, '..', 'vm'),
'--dy',
'graalpython',
]
if not DISABLE_REBUILD:
# build GraalPy and all the necessary dependencies, so that we can deploy them
run_mx(run_mx_args + ["build"], env={**env, **LATEST_JAVA_HOME})
# deploy maven artifacts
version = GRAAL_VERSION
path = os.path.join(SUITE.get_mx_output_dir(), 'public-maven-repo')
licenses = ['EPL-2.0', 'PSF-License', 'GPLv2-CPE', 'ICU,GPLv2', 'BSD-simplified', 'BSD-new', 'UPL', 'MIT', 'GFTC']
deploy_args = run_mx_args + [
'maven-deploy',
'--tags=public',
'--all-suites',
'--all-distribution-types',
f'--version-string={version}',
'--validate=none',
'--licenses', ','.join(licenses),
'--suppress-javadoc',
'local',
pathlib.Path(path).as_uri(),
]
if not DISABLE_REBUILD:
mx.rmtree(path, ignore_errors=True)
os.mkdir(path)
run_mx(deploy_args, env={**env, **LATEST_JAVA_HOME})
repo_url = pathlib.Path(path).as_uri()
for library_name in ('BOUNCYCASTLE-PROVIDER', 'BOUNCYCASTLE-PKIX', 'BOUNCYCASTLE-UTIL'):
deploy_library_to_local_maven_repo(library_name, repo_url, {**env, **LATEST_JAVA_HOME})
return path, version, env
def deploy_graalpy_extensions_to_local_maven_repo(env=None, only_projects=None):
env = update_maven_opts({**os.environ.copy(), **(env or {})})
env["MVNW_REPOURL"] = mx_urlrewrites.rewriteurl("https://repo.maven.apache.org/maven2/").rstrip('/')
env["MVNW_VERBOSE"] = "true"
gradle_java_home = os.environ.get('GRADLE_JAVA_HOME')
if not gradle_java_home:
def abortCallback(msg):
mx.abort("Could not find a JDK of version between 17 and 21 to build a Gradle plugin from graalpy-extensions.\n"
"Export GRADLE_JAVA_HOME pointing to a suitable JDK or use the generic MX mechanism explained below:\n" + msg)
gradle_java_home = mx.get_tools_jdk('17..21', abortCallback=abortCallback).home
graalpy_extensions_path = os.environ.get('GRAALPY_EXTENSIONS_PATH')
if not graalpy_extensions_path:
mx.log("Cloning graalpy-extensions. If you want to use custom local clone, set env variable GRAALPY_EXTENSIONS_PATH")
graalpy_extensions_path = os.path.join(SUITE.get_mx_output_dir(), 'graalpy-extensions')
if os.path.exists(graalpy_extensions_path):
shutil.rmtree(graalpy_extensions_path)
mx.run(['git', 'clone', '--depth=1', mx_urlrewrites.rewriteurl('https://github.com/oracle/graalpy-extensions.git'), graalpy_extensions_path])
local_repo_path = os.path.join(SUITE.get_mx_output_dir(), 'public-maven-repo')
# setup symlink .mvn/maven-bundle -> local repo path
maven_dir = os.path.join(graalpy_extensions_path, '.mvn')
bundle_path = os.path.join(maven_dir, 'maven-bundle')
if os.path.lexists(bundle_path):
mx.abort(f"Refusing to override existing '{bundle_path}' when building graalpy-extensions.")
os.makedirs(maven_dir, exist_ok=True)
try:
os.symlink(os.path.abspath(local_repo_path), bundle_path, target_is_directory=True)
except OSError as e:
mx.abort(f"Could not create {bundle_path} -> {local_repo_path}: {e}")
version = GRAAL_VERSION
common_args = [
'-DskipJavainterfacegen',
'-DskipSigtest',
'-DskipTests',
f'-Drevision={version}',
f'-Dlocal.repo.url=' + pathlib.Path(local_repo_path).as_uri(),
f"-Dgradle.java.home={gradle_java_home}"
]
with use_local_maven_repo_settings(local_repo_path) as maven_settings:
mx.run([os.path.join(graalpy_extensions_path, mx.cmd_suffix('mvnw')),
'--settings', maven_settings,
*common_args, '-Pmxurlrewrite',
'-N', 'exec:java@patch-gradle-props'],
env=env, cwd=graalpy_extensions_path)
if only_projects:
common_args += ['-pl', ','.join(only_projects)]
mx.run([os.path.join(graalpy_extensions_path, mx.cmd_suffix('mvnw')),
'--settings', maven_settings,
*common_args, '-DdeployAtEnd=true',
f'-DaltDeploymentRepository=local::{pathlib.Path(local_repo_path).as_uri()}',
'deploy'], env=env, cwd=graalpy_extensions_path)
return local_repo_path, version, env
def deploy_graalpy_extensions_to_local_maven_repo_wrapper(*_):
deploy_graalpy_extensions_to_local_maven_repo()
def deploy_local_maven_repo_wrapper(args):
p, _, _ = deploy_local_maven_repo()
if '--with-extensions' in args:
deploy_graalpy_extensions_to_local_maven_repo()
print(f"local Maven repo path: {p}")
def python_jvm(_=None):
"""Returns the path to GraalPy from 'jvm' standalone dev build. Also builds the standalone."""
launcher = graalpy_standalone('jvm', dev=True)
mx.log(launcher)
return launcher
def python_gvm(_=None):
"""Deprecated, use python-jvm"""
mx.warn("mx python-gvm and the helper function python_gvm are deprecated, use python-jvm/python_jvm")
return python_jvm()
def make_coverage_launcher_if_needed(launcher):
if mx_gate.get_jacoco_agent_args():
# patch our launchers created under jacoco to also run with jacoco.
# do not use is_collecting_coverage() here, we only want to patch when
# jacoco agent is requested.
quote = shlex.quote if sys.platform != 'win32' else lambda x: x
def graalvm_vm_arg(java_arg):
return quote(f'--vm.{java_arg[1:] if java_arg.startswith("-") else java_arg}')
agent_args = ' '.join(graalvm_vm_arg(arg) for arg in mx_gate.get_jacoco_agent_args() or [])
# We need to make sure the arguments get passed to subprocesses, so we create a temporary launcher
# with the arguments.
original_launcher = os.path.abspath(os.path.realpath(launcher))
if sys.platform != 'win32':
coverage_launcher = original_launcher + "_cov"
c_launcher_source = coverage_launcher + ".c"
agent_args_list = shlex.split(agent_args)
extra_args_c = []
for arg in agent_args_list:
extra_args_c.append('new_args[arg_index++] = "' + arg.replace("\"", r"\"") + '";')
extra_args_c = ' '.join(extra_args_c)
c_code = dedent(f"""\
#include
#include
#include
int main(int argc, char **argv) {{
char *new_args[argc + 3 + {len(agent_args_list)}];
int arg_index = 0;
new_args[arg_index++] = argv[0];
new_args[arg_index++] = "--jvm";
{extra_args_c}
for (int i = 1; i < argc; i++) {{
new_args[arg_index++] = argv[i];
}}
new_args[arg_index] = NULL;
execvp("{original_launcher}", new_args);
perror("execvp failed");
return 1;
}}
""")
with open(c_launcher_source, "w") as f:
f.write(c_code)
compile_cmd = ["cc", c_launcher_source, "-o", coverage_launcher]
subprocess.check_call(compile_cmd)
os.chmod(coverage_launcher, 0o775)
else:
coverage_launcher = original_launcher.replace('.exe', '.cmd')
# Windows looks for libraries on PATH, we need to add the jvm bin dir there or it won't find the instrumentation dlls
jvm_bindir = os.path.join(os.path.dirname(os.path.dirname(original_launcher)), 'jvm', 'bin')
with open(coverage_launcher, "w") as f:
f.write(f'@echo off\nset PATH=%PATH%;{jvm_bindir}\n')
exe_arg = quote(f"--python.Executable={coverage_launcher}")
f.write(f'{original_launcher} --jvm {exe_arg} {agent_args} %*\n')
mx.log(f"Replaced {launcher} with {coverage_launcher} to collect coverage")
launcher = coverage_launcher
return launcher
def python_svm(_=None):
"""Returns the path to GraalPy native image from 'native' standalone dev build.
Also builds the standalone if not built already."""
if mx_gate.get_jacoco_agent_args():
return python_jvm()
launcher = graalpy_standalone('native')
mx.log(launcher)
return launcher
def _python_test_runner():
return os.path.join(SUITE.dir, "graalpython", "com.oracle.graal.python.test", "src", "runner.py")
def _python_unittest_root():
return os.path.join(SUITE.dir, "graalpython", "com.oracle.graal.python.test", "src", "tests")
def graalpytest(args):
# help is delegated to the runner, it will fake the mx-specific options as well
parser = ArgumentParser(prog='mx graalpytest', add_help=False)
parser.add_argument('--python')
parser.add_argument('--svm', action='store_true')
args, unknown_args = parser.parse_known_args(args)
env = extend_os_env(
MX_GRAALPYTEST='1',
PYTHONHASHSEED='0',
)
python_args = []
runner_args = []
for arg in unknown_args:
if arg.startswith(('--python.', '--engine.', '--vm.', '--inspect', '--log.', '--experimental-options', '-multi-context', '-repeated-run')):
python_args.append(arg)
else:
runner_args.append(arg)
# if we got a binary path it's most likely CPython, so don't add graalpython args
is_graalpy = False
python_binary = args.python
if not python_binary:
is_graalpy = True
python_args = ["--experimental-options=true", "--python.EnableDebuggingBuiltins", *python_args]
if args.svm:
python_binary = graalpy_standalone_native()
elif 'graalpy' in os.path.basename(python_binary) or 'mxbuild' in python_binary:
is_graalpy = True
gp_args = ["--experimental-options=true", "--python.EnableDebuggingBuiltins"]
if env.get("GRAALPYTEST_ALLOW_NO_JAVA_ASSERTIONS") != "true":
gp_args += ["--vm.ea", "--vm.esa"]
mx.log(f"Executable seems to be GraalPy, prepending arguments: {gp_args}")
python_args = [*gp_args, *python_args]
runner_args.append(f'--subprocess-args={shlex.join(arg for arg in python_args if arg != "-repeated-run")}')
if is_graalpy:
runner_args.append(f'--append-path={os.path.join(_dev_pythonhome(), "lib-python", "3")}')
cmd_args = [*python_args, _python_test_runner(), 'run', *runner_args]
delete_bad_env_keys(env)
if python_binary:
return run([python_binary, *cmd_args], nonZeroIsFatal=True, env=env)
else:
return full_python(cmd_args, env=env)
def run_python_unittests(python_binary, args=None, paths=None, exclude=None, env=None,
cwd=None, lock=None, out=None, err=None, nonZeroIsFatal=True, timeout=None,
report: Union[Task, bool, None] = False, parallel=None, runner_args=None, test_runner=None,
reportfile=None, runner_reportfile=None):
if lock:
lock.acquire()
if parallel is None:
parallel = 4 if paths is None else 1
if sys.platform == 'win32':
if CI:
# Windows machines don't seem to have much memory
parallel = min(parallel, 2)
if GITHUB_CI:
parallel = 0
if mx_gate.get_jacoco_agent_args():
# JaCoCo execution data is appended to a shared file and cannot be
# safely written by multiple instrumented JVM test workers at once.
parallel = 1
parallelism = str(min(os.cpu_count() or 1, parallel))
args = args or []
extra_args = shlex.split(os.environ.get("GRAALPY_UNITTEST_ARGS", ""))
if extra_args:
mx.log("Adding GraalPy unittest args from GRAALPY_UNITTEST_ARGS: " + shlex.join(extra_args))
args = [
"--vm.ea",
"--experimental-options=true",
"--python.EnableDebuggingBuiltins",
*args,
*extra_args,
]
if env is None:
env = os.environ.copy()
env['PYTHONHASHSEED'] = '0'
delete_bad_env_keys(env)
if mx.primary_suite() != SUITE:
env.setdefault("GRAALPYTEST_ALLOW_NO_JAVA_ASSERTIONS", "true")
if (pip_index := env.get("PIP_INDEX_URL")) and "PIP_EXTRA_INDEX_URL" not in env:
# the user was overriding the index, don't sneak our default extra
# index in in that case
env["PIP_EXTRA_INDEX_URL"] = pip_index
args += [test_runner or _python_test_runner(), "run", "--durations", "10", "-n", parallelism, f"--subprocess-args={shlex.join(args)}"]
if runner_args:
args += runner_args
if exclude:
for file in exclude:
args += ['--ignore', file]
if is_collecting_coverage() and mx_gate.get_jacoco_agent_args():
# jacoco only dumps the data on exit, and when we run all our unittests
# at once it generates so much data we run out of heap space
args.append('--separate-workers')
t0 = time.time()
if report:
if reportfile is None:
with tempfile.NamedTemporaryFile(prefix="test-report-", suffix=".json", delete=False) as report_tmp:
reportfile = os.path.abspath(report_tmp.name)
else:
reportfile = os.path.abspath(reportfile)
args += ["--mx-report", runner_reportfile or reportfile]
if paths is not None:
args += paths
else:
args.append(os.path.relpath(_python_unittest_root()))
mx.logv(shlex.join([python_binary] + args))
if lock:
lock.release()
result = run([python_binary] + args, nonZeroIsFatal=nonZeroIsFatal, env=env, cwd=cwd, out=out, err=err, timeout=timeout)
if lock:
lock.acquire()
if isinstance(report, mx.Task):
if reportfile:
mx_gate.make_test_report(reportfile, report.title)
else:
mx_gate.make_test_report([{
"name": report.title,
"status": "PASSED" if result == 0 else "FAILED",
"duration": int((time.time() - t0) * 1000)
}], report.title)
if lock:
lock.release()
return result
def run_sandboxed_tests(python_binary, report, args=None, **kwargs):
args = SANDBOXED_OPTIONS + (args or [])
exclude = list(kwargs.pop("exclude", []) or [])
if not HAS_JEP_454:
exclude += [
"cpyext",
"test_ctypes",
"test_ctypes_callbacks",
]
env = dict(kwargs.pop("env", os.environ.copy()) or {})
propagated_args = [
"--experimental-options=true",
*[arg for arg in args if arg.startswith(("--python.", "--vm.", "--experimental-options"))],
]
env["GRAAL_PYTHON_VM_ARGS"] = "\v" + "\v".join(propagated_args)
run_python_unittests(python_binary, args=args, env=env, report=report, exclude=exclude, **kwargs)
# TODO the test runner doesn't even find the tests on Darwin
if sys.platform == "darwin":
return
tagged_test_path = os.path.relpath(os.path.join(_get_stdlib_home(), 'test'))
tagged_tests = [
# compression
'test_zlib.py',
# TODO
# 'test_lzma.py',
# 'test_zipimport.py',
# sha3
'test_hashlib.py',
# expat
'test_pyexpat.py',
'test_xml_etree.py',
'test_xml_dom_minicompat.py',
'test_sax.py',
'test_pulldom.py',
'test_minidom.py',
]
paths = [os.path.join(tagged_test_path, test) for test in tagged_tests]
run_tagged_unittests(python_binary, args=args, paths=paths, report=report, **kwargs)
def run_tagged_unittests(python_binary, env=None, cwd=None, nonZeroIsFatal=True, checkIfWithGraalPythonEE=False,
report: Union[Task, bool, None] = False, parallel=8, exclude=None, paths=(), args=None):
if checkIfWithGraalPythonEE:
mx.run([python_binary, "-c", "import sys; print(sys.version)"])
run_python_unittests(
python_binary,
args=args,
runner_args=[f'--append-path={os.path.join(_dev_pythonhome(), "lib-python", "3")}', '--tagged'],
paths=paths or [os.path.relpath(os.path.join(_get_stdlib_home(), 'test'))],
env=env,
cwd=cwd,
nonZeroIsFatal=nonZeroIsFatal,
report=report,
parallel=parallel,
exclude=exclude,
)
def get_cpython():
if python3_home := os.environ.get("PYTHON3_HOME"):
return os.path.join(python3_home, "python")
else:
return "python3"
def get_wrapper_urls(wrapper_properties_file, keys):
ret = dict()
with(open(wrapper_properties_file)) as f:
while line := f.readline():
line = line.strip()
for key in keys:
if not line.startswith("#") and key not in ret.keys() and key in line:
s = line.split("=")
if len(s) > 1:
ret.update({key : mx_urlrewrites.rewriteurl(s[1].strip())})
break
for key in keys:
assert key in ret.keys(), f"Expected key '{key}' to be in {wrapper_properties_file}, but was not."
return ret
def graalpython_gate_runner(_, tasks):
report = lambda: (not is_collecting_coverage()) and task
nonZeroIsFatal = not is_collecting_coverage()
# JUnit tests
with Task('GraalPython JUnit', tasks, tags=[GraalPythonTags.junit]) as task:
if task:
run_mx(["build"], env={**os.environ, **LATEST_JAVA_HOME})
if WIN32:
punittest(
[
"--verbose",
"--no-leak-tests",
"--regex",
r'((graal\.python\.test\.integration)|(graal\.python\.test\.(builtin|interop|util))|(graal\.python\.cext\.test))'
],
report=True
)
else:
punittest(['--verbose'], report=report())
# Run tests with static exclusion paths
jdk = mx.get_jdk()
prev = jdk.java_args_pfx
try:
java_args = shlex.split(mx._opts.java_args) if mx._opts.java_args else []
jdk.java_args_pfx = java_args + ['-Dpython.WithoutPlatformAccess=true']
punittest(['--verbose', '--no-leak-tests', '--regex', 'com.oracle.graal.python.test.advanced.ExclusionsTest'])
finally:
jdk.java_args_pfx = prev
if report():
tmpfile = tempfile.NamedTemporaryFile(delete=False, suffix='.json.gz')
try:
# Cannot use context manager because windows doesn't allow
# make_test_report to read the file while it is open for
# writing
mx.command_function('tck')([f'--json-results={tmpfile.name}'])
mx_gate.make_test_report(tmpfile.name, GraalPythonTags.junit + "-TCK")
finally:
tmpfile.close()
try:
os.unlink(tmpfile.name)
except:
pass # Sometimes this fails on windows
else:
mx.command_function('tck')([])
verify_junit_compilation_failure()
# JUnit tests with Maven
with Task('GraalPython integration JUnit with Maven', tasks, tags=[GraalPythonTags.junit_maven]) as task:
if task:
mvn_repo_path, artifacts_version, env = deploy_local_maven_repo()
pom_path = os.path.join(SUITE.dir, 'graalpython/com.oracle.graal.python.test.integration/pom.xml')
env['PATH'] = get_path_with_patchelf()
with use_local_maven_repo_settings(mvn_repo_path) as maven_settings:
mvn_cmd_base = ['--settings', maven_settings,
'-f', pom_path,
f'-Dcom.oracle.graal.python.test.polyglot.version={artifacts_version}',
'--batch-mode']
default_runtime_arg = []
if not HAS_JEP_454:
# Bytecode DSL does not work on JDK21 with optimizing runtime (GR-72424)
default_runtime_arg = ['-Dtruffle.UseFallbackRuntime=true']
graalvm_jdk_path = graalvm_jdk()
mx.log(f"Running integration JUnit tests on GraalVM SDK: {graalvm_jdk_path} (extra arguments: {' '.join(default_runtime_arg)})")
env['JAVA_HOME'] = graalvm_jdk_path
mx.run_maven(mvn_cmd_base + [*default_runtime_arg, '-U', 'clean', 'test'], env=env)
env['JAVA_HOME'] = os.environ['JAVA_HOME']
mx.log(f"Running integration JUnit tests on vanilla JDK: {os.environ.get('JAVA_HOME', 'system java')}")
mx.run_maven(mvn_cmd_base + ['-U', '-Dpolyglot.engine.WarnInterpreterOnly=false', 'clean', 'test'], env=env)
# JUnit tests with Maven and polyglot isolates
with Task('GraalPython integration JUnit with Maven and Polyglot Isolates', tasks, tags=[GraalPythonTags.junit_maven_isolates]) as task:
if task:
if mx.is_windows():
mx.log(mx.colorize('Polyglot isolate tests do not work on Windows', color='magenta'))
return
mvn_repo_path, artifacts_version, env = deploy_local_maven_repo(env={
"DYNAMIC_IMPORTS": "/truffle-enterprise,/substratevm-enterprise",
"NATIVE_IMAGES": "",
"POLYGLOT_ISOLATES": "python",
})
pom_path = os.path.join(SUITE.dir, 'graalpython/com.oracle.graal.python.test.integration/pom.xml')
env['PATH'] = get_path_with_patchelf()
with use_local_maven_repo_settings(mvn_repo_path) as maven_settings:
mvn_cmd_base = ['--settings', maven_settings,
'-f', pom_path,
f'-Dcom.oracle.graal.python.test.polyglot.version={artifacts_version}',
'--batch-mode']
mx.log("Running integration JUnit tests on GraalVM SDK with external polyglot isolates")
env['JAVA_HOME'] = graalvm_jdk(enterprise=True)
mx.run_maven(mvn_cmd_base + [
'-U',
'-Pisolate',
'-Dpolyglot.engine.AllowExperimentalOptions=true',
'-Dpolyglot.engine.SpawnIsolate=true',
'-Dpolyglot.engine.IsolateMode=external',
'-Dpolyglot.engine.WarnMethodScoping=false',
'clean',
'test',
], env=env)
mx.log("Running integration JUnit tests on GraalVM SDK with untrusted sandbox policy")
mx.run_maven(mvn_cmd_base + [
'-Pisolate',
'-Dtest=SandboxPolicyUntrustedTest',
'test',
], env=env)
# Unittests on JVM
with Task('GraalPython JVM build', tasks, tags=[GraalPythonTags.jvmbuild]) as task:
if task:
graalpy_standalone_jvm()
with Task('GraalPython Python unittests', tasks, tags=[GraalPythonTags.unittest]) as task:
if task:
run_python_unittests(
graalpy_standalone_jvm(),
nonZeroIsFatal=nonZeroIsFatal,
report=report(),
parallel=6,
)
with Task('GraalPython BouncyCastle unittests', tasks, tags=[GraalPythonTags.unittest_bouncycastle]) as task:
if task:
bc_unit_tests = [
f"graalpython/com.oracle.graal.python.test/src/tests/test_ssl.py::tests.test_ssl.CertTests.test_{t}"
for t in (
"private_key_pkcs1_password",
"private_key_ec_legacy",
"private_key_dsa_legacy",
"verify_x509_strict",
)
]
bc_stdlib_tests = [
f"graalpython/lib-python/3/test/test_hashlib.py::test.test_hashlib.HashLibTestCase.test_{t}"
for t in (
"algorithms_available",
"blocksize_name_blake2",
"blocksize_name_sha3",
"case_blake2b_0",
"case_blake2b_1",
"case_blake2s_0",
"case_blake2s_1",
"clinic_signature",
"digest_length_overflow",
"gil",
"hash_array",
"hexdigest",
"large_update",
"name_attribute",
"usedforsecurity_false",
"usedforsecurity_true",
)
] + [
"graalpython/lib-python/3/test/test_ssl.py::test.test_ssl.ThreadedTests.test_verify_strict",
]
assert GRAALPY_WITH_BOUNCYCASTLE, "This gate only makes sense when bouncycastle was enabled using the envvar GRAALPY_WITH_BOUNCYCASTLE"
graalpy = graalpy_standalone_jvm()
run_python_unittests(graalpy, paths=bc_unit_tests, report=report())
stdlib_test_args = [f'--append-path={os.path.join(_dev_pythonhome(), "lib-python", "3")}', '--all']
run_python_unittests(graalpy, paths=bc_stdlib_tests, runner_args=stdlib_test_args, report=report())
with Task('GraalPython Python unittests with CPython', tasks, tags=[GraalPythonTags.unittest_cpython]) as task:
if task:
env = extend_os_env(PYTHONHASHSEED='0')
test_args = [get_cpython(), _python_test_runner(), "run", "-n", "6", "graalpython/com.oracle.graal.python.test/src/tests"]
run(test_args, nonZeroIsFatal=True, env=env)
with Task('GraalPython sandboxed tests', tasks, tags=[GraalPythonTags.unittest_sandboxed]) as task:
if task:
run_sandboxed_tests(graalpy_standalone_jvm(), report=report())
with Task('GraalPython sandboxed tests on SVM', tasks, tags=[GraalPythonTags.svmunit_sandboxed]) as task:
if task:
run_sandboxed_tests(graalpy_standalone_native_enterprise(), parallel=8, report=report())
with Task('GraalPython multi-context unittests', tasks, tags=[GraalPythonTags.unittest_multi]) as task:
if task:
env = os.environ.copy()
graalpy = graalpy_standalone_native()
env['PATH'] = get_path_with_patchelf()
mx.log("1. Running twice without shared engine")
run_python_unittests(
graalpy,
args=["-repeated-run", "--python.IsolateNativeModules=true"],
parallel=0,
exclude=MULTI_CONTEXT_EXCLUSIONS,
env=env,
nonZeroIsFatal=nonZeroIsFatal,
report=report(),
)
mx.log("2. Running twice with shared engine")
run_python_unittests(
graalpy,
args=["-repeated-run", "-multi-context", "--python.IsolateNativeModules=true"],
parallel=0,
exclude=MULTI_CONTEXT_EXCLUSIONS,
env=env,
nonZeroIsFatal=nonZeroIsFatal,
report=report(),
)
with Task('GraalPython sandboxed multi-context tests', tasks, tags=[GraalPythonTags.unittest_multi_sandboxed]) as task:
if task:
mx.log("Running twice with shared engine")
run_sandboxed_tests(
graalpy_standalone_jvm(),
args=["-repeated-run", "-multi-context"],
parallel=0,
exclude=MULTI_CONTEXT_EXCLUSIONS,
nonZeroIsFatal=nonZeroIsFatal,
report=report(),
)
with Task('GraalPython Jython emulation tests', tasks, tags=[GraalPythonTags.unittest_jython]) as task:
if task:
run_python_unittests(graalpy_standalone_jvm(), args=["--python.EmulateJython"], paths=["test_interop.py"], report=report(), nonZeroIsFatal=nonZeroIsFatal)
with Task('GraalPython with Arrow Storage Strategy', tasks, tags=[GraalPythonTags.unittest_arrow]) as task:
if task:
run_python_unittests(graalpy_standalone_jvm(), args=["--python.UseNativePrimitiveStorageStrategy"], report=report(), nonZeroIsFatal=nonZeroIsFatal)
with Task('GraalPython standalone module tests', tasks, tags=[GraalPythonTags.unittest_standalone]) as task:
if task:
gvm_jdk = graalvm_jdk()
standalone_home = graalpy_standalone_home('jvm')
mvn_repo_path, version, env = deploy_local_maven_repo()
deploy_graalpy_extensions_to_local_maven_repo(only_projects=['org.graalvm.python.embedding'])
if RUNNING_ON_LATEST_JAVA:
# our standalone python binary is meant for standalone graalpy
# releases which are only for latest
env['ENABLE_STANDALONE_UNITTESTS'] = 'true'
env['JAVA_HOME'] = gvm_jdk
env['PYTHON_STANDALONE_HOME'] = standalone_home
env['GRAAL_VERSION'] = version
# setup maven downloader overrides
env['MAVEN_REPO_OVERRIDE'] = ",".join([
f"{pathlib.Path(mvn_repo_path).as_uri()}/",
mx_urlrewrites.rewriteurl('https://repo1.maven.org/maven2/'),
])
env["org.graalvm.maven.downloader.version"] = version
env["org.graalvm.maven.downloader.repository"] = f"{pathlib.Path(mvn_repo_path).as_uri()}/"
# run the test
mx.logv(f"running with os.environ extended with: {env=}")
run_python_unittests(
os.path.join(standalone_home, 'bin', _graalpy_launcher()),
paths=["graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py"],
env=env,
parallel=3,
)
with Task('GraalPython Python tests', tasks, tags=[GraalPythonTags.tagged]) as task:
if task:
# don't fail this task if we're running with the jacoco agent, we know that some tests don't pass with it enabled
collecting_coverage = is_collecting_coverage()
run_tagged_unittests(
graalpy_standalone_native(),
# GR-78213: Temporarily work around transient failures in test_gzip
args=[] if collecting_coverage else ["--engine.CompileOnly=~_PaddedFile.read"],
nonZeroIsFatal=not collecting_coverage,
report=report(),
)
# Unittests on SVM
with Task('GraalPython build on SVM', tasks, tags=[GraalPythonTags.svmbuild]) as task:
if task:
graalpy_standalone_native()
with Task('GraalPython GraalOS standalone build on SVM', tasks, tags=[GraalPythonTags.svm_graalos_standalone_build]) as task:
if task:
branch = _normalize_branch_name(os.environ.get("TO_BRANCH") or SUITE.vc.active_branch(SUITE.dir, abortOnError=False) or 'master')
if branch == 'master':
on_fail = mx.abort
else:
def on_fail(codeOrMessage, context=None, killsig=signal.SIGTERM):
mx.warn(codeOrMessage, context=context)
assert False, f"GraalOS build and test failed, signal {killsig} ignored"
try:
mx_graalpython_graalos.graalpy_graalos_standalone_build_and_test(report=report(), on_fail=on_fail)
except AssertionError:
pass
with Task('GraalPython tests on SVM', tasks, tags=[GraalPythonTags.svmunit]) as task:
if task:
run_python_unittests(graalpy_standalone_native(), parallel=8, report=report())
with Task('GraalPython license header update', tasks, tags=[GraalPythonTags.license]) as task:
if task:
python_checkcopyrights([])
with Task('GraalPython GraalVM build', tasks, tags=[GraalPythonTags.svm, GraalPythonTags.graalvm], report=True) as task:
if task:
with set_env(PYTHONIOENCODING=None, MX_CHECK_IOENCODING="0"):
svm_image = python_svm()
benchmark = os.path.join(PATH_MESO, "image-magix.py")
out = mx.OutputCapture()
run([svm_image, "-S", "--log.python.level=FINE", benchmark], nonZeroIsFatal=True, out=mx.TeeOutputCapture(out), err=mx.TeeOutputCapture(out))
success = "\n".join([
"[0, 0, 0, 0, 0, 0, 10, 10, 10, 0, 0, 10, 3, 10, 0, 0, 10, 10, 10, 0, 0, 0, 0, 0, 0]",
])
if success not in out.data:
mx.abort('Output from generated SVM image "' + svm_image + '" did not match success pattern:\n' + success)
if not WIN32:
assert "Using preinitialized context." in out.data
with Task('Python SVM Truffle TCK', tasks, tags=[GraalPythonTags.language_checker], report=True) as task:
if task:
run_mx([
"--dy", "graalpython,/substratevm",
"-p", os.path.join(mx.suite("truffle").dir, "..", "vm"),
"--native-images=",
"build",
], env={**os.environ, **LATEST_JAVA_HOME})
run_mx([
"--dy", "graalpython,/substratevm",
"-p", os.path.join(mx.suite("truffle").dir, "..", "vm"),
"--native-images=",
"gate", "svm-truffle-tck-python",
])
with Task("Graalpython tox example", tasks, tags=["tox-example"]) as task:
if task:
try:
tox_example([])
except:
mx.log("TIP: run 'mx help tox-example' to learn more about reproducing this test locally")
raise
if WIN32 and is_collecting_coverage():
mx.log("Ask for shutdown of any remaining graalpy.exe processes")
# On windows, the jacoco command can fail if the file is still locked
# by lingering test processes, so we try to give it a bit of a cleanup
mx.run([
'taskkill.exe',
'/T', # with children
'/IM',
'graalpy.exe',
], nonZeroIsFatal=False)
time.sleep(2)
mx.log("Forcefully terminate any remaining graalpy.exe processes")
mx.run([
'taskkill.exe',
'/F', # force
'/T', # with children
'/IM',
'graalpy.exe',
], nonZeroIsFatal=False)
# Forcefully killing processes on Windows does not release file
# locks immediately, so we still need to sleep for a bit in the
# hopes that the OS will release
time.sleep(8)
mx_gate.add_gate_runner(SUITE, graalpython_gate_runner)
def tox_example(args=None):
"""
Runs the tox example: executing tox in a CPython venv, which then executes
pytest tests of an example package 'leftpad' on GraalPython.
To pass additional arguments to GraalPython, set the GRAAL_PYTHON_ARGS
environment variable, tox will forward it to GraalPython.
Run with '--help' to learn about supported options.
"""
import argparse
parser = argparse.ArgumentParser(prog='mx tox-example')
parser.add_argument("--reuse-venv", action="store_true",
help="Whether to reuse existing venv created by previous invocations of this command.")
opts = parser.parse_args(args)
graalpy = graalpy_standalone_native_enterprise()
tox_project_dir = os.path.join(
cast(mx.Project, mx.project("com.oracle.graal.python.test", fatalIfMissing=True)).dir,
"src",
"tox"
)
mx.log("Setting up CPython venv to run tox itself")
libs = [
"distlib==0.4.3",
"filelock==3.32.2",
"packaging==26.2",
"platformdirs==4.11.0",
"pluggy==1.6.0",
"pyparsing==3.3.2",
"six==1.17.0",
"toml==0.10.2",
"tox==4.58.0",
"virtualenv==21.7.1",
os.path.join(os.path.dirname(graalpy), "..", "graalpy_virtualenv_seeder"),
]
def get_new_vm(project_name, install_libs=None, reuse_existing=False):
if install_libs is None:
install_libs = []
import platform
mx.log("[platform] {}".format(platform.uname()))
path = os.path.join(mx.dependency("com.oracle.graal.python.test").get_output_root(), "tox_venv")
reuse = os.path.exists(path) and reuse_existing
action_name = "Reusing existing" if reuse else "Creating"
mx.logv("{} venv for {} in {}".format(action_name, project_name, path))
# remove any pre-existing venv to ensure that launchers are freshly created
if not reuse and os.path.isdir(path):
mx.log("Deleting pre-existing venv in {}".format(path))
from shutil import rmtree
rmtree(path)
quiet_opt = ["-q"] if not mx._opts.verbose else []
env_py3_home = os.environ.get("PYTHON3_HOME")
if env_py3_home:
python = os.path.join(env_py3_home, "python3")
mx.logv("Overriding 'python3' using environment variable PYTHON3_HOME to '{}'".format(python))
else:
python = "python3"
mx.log("{} CPython venv for {} (bin: {})".format(action_name, project_name, python))
if not reuse:
mx.run([python, "-m", "venv", "--clear", path])
vm = os.path.join(path, "bin", "python")
os.environ['VIRTUAL_ENV'] = path
os.environ['PATH'] = "{}:{}".format(os.path.join(path, "bin"), os.environ.get("PATH", ""))
if not reuse:
for lib in install_libs:
try:
cmd = [vm, "-m", "pip"] + quiet_opt + ["install", lib]
mx.log("running: {}".format(' '.join(cmd)))
mx.run(cmd)
except:
mx.abort("Could not install dependency %s" % install_libs)
os.environ['PYTHON'] = vm
os.environ["PYTHON_VM"] = vm
return vm
python3 = get_new_vm("tox", install_libs=libs, reuse_existing=opts.reuse_venv)
new_env = os.environ.copy()
new_env['PATH'] = new_env['PATH'] + os.pathsep + os.path.dirname(graalpy)
mx.log(f"Added {graalpy} to the PATH")
def check_output(expected, lines):
for e in expected:
if not any(e in l for l in lines):
mx.abort("Could not find expected {} in the output".format(e))
# Passing tests:
mx.log("Running {} -m tox -e graalpy".format(python3))
wd = os.path.join(tox_project_dir, "leftpad")
output = mx.LinesOutputCapture()
mx.log("Running {} -m tox -e graalpy".format(python3))
output_capture = mx.TeeOutputCapture(output)
mx.run([python3, "-m", "tox"], env=new_env, cwd=wd, out=output_capture, err=output_capture)
check_output(["4 passed", "graalpy: OK"], output.lines)
# Failing tests:
mx.log("Running {} -m tox -e graalpy with intentionally failing tests".format(python3))
output = mx.LinesOutputCapture()
new_env['GRAALPY_LEFTPAD_FAIL'] = '1'
output_capture = mx.TeeOutputCapture(output)
exit_code = mx.run([python3, "-m", "tox"], env=new_env, cwd=wd, out=output_capture, err=output_capture, nonZeroIsFatal=False)
check_output(["test_leftpad.py::test_leftpad_failing - AssertionError", "1 failed, 3 passed"], output.lines)
if exit_code == 0:
mx.abort("Expected the tests to fail")
class ArchiveProject(mx.ArchivableProject):
def __init__(self, suite, name, deps, workingSets, theLicense, **_):
super(ArchiveProject, self).__init__(suite, name, deps, workingSets, theLicense)
def output_dir(self):
if hasattr(self, "outputFile"):
self.outputFile = mx_subst.path_substitutions.substitute(self.outputFile)
return os.path.dirname(os.path.join(self.dir, self.outputFile))
else:
assert hasattr(self, "outputDir")
self.outputDir = mx_subst.path_substitutions.substitute(self.outputDir)
return os.path.join(self.dir, self.outputDir)
def archive_prefix(self):
return mx_subst.path_substitutions.substitute(getattr(self, "prefix", ""))
def getArchivableResults(self, use_relpath=True, single=False):
for f, arcname in super().getArchivableResults(use_relpath=use_relpath, single=single):
yield f, arcname.replace(os.sep, "/")
def getResults(self):
if hasattr(self, "outputFile"):
return [os.path.join(self.dir, self.outputFile)]
else:
ignore_regexps = [re.compile(s) for s in getattr(self, "ignorePatterns", [])]
results = []
for root, _, files in os.walk(self.output_dir()):
for name in files:
path = os.path.join(root, name)
if not any(r.search(path) for r in ignore_regexps):
results.append(path)
return results
def deploy_binary_if_main(args):
"""if the active branch is the main branch, deploy binaries for the primary suite to remote maven repository."""
active_branch = SUITE.vc.active_branch(SUITE.dir)
if active_branch == MAIN_BRANCH:
if sys.platform == "darwin":
args.insert(0, "--platform-dependent")
return mx.command_function('deploy-binary')(args)
else:
mx.log('The active branch is "%s". Binaries are deployed only if the active branch is "%s".' % (
active_branch, MAIN_BRANCH))
return 0
def _get_suite_dir(suitename):
return mx.suite(suitename).dir
def _get_suite_parent_dir(suitename):
return os.path.dirname(mx.suite(suitename).dir)
def _get_src_dir(projectname):
for suite in mx.suites():
for p in suite.projects:
if p.name == projectname:
if len(p.source_dirs()) > 0:
return p.source_dirs()[0]
else:
return p.dir
mx.abort("Could not find src dir for project %s" % projectname)
def _get_output_root(projectname):
prefix, _, suffix = projectname.rpartition(":")
for suite in mx.suites():
if prefix and suite.name != prefix:
continue
for p in itertools.chain(suite.projects, suite.dists):
if p.name == suffix:
try:
return p.get_output_root()
except:
return p.get_output()
mx.abort("Could not find out dir for project %s" % projectname)
# We use the ordinal value of this character and add it to the version parts to
# ensure that we store ASCII-compatible printable characters into the versions
# file.
#
# IMPORTANT: This needs to be in sync with 'PythonLanguage.VERSION_BASE' and
# 'PythonResource.VERSION_BASE'.
VERSION_BASE = '!'
def py_version_short(variant=None, **_):
if variant == 'major_minor_nodot':
return PYTHON_VERSION_MAJ_MIN.replace(".", "")
elif variant == 'binary':
return "".join([chr(int(p) + ord(VERSION_BASE)) for p in PYTHON_VERSION.split(".")])
else:
return PYTHON_VERSION_MAJ_MIN
def graal_version_short(variant=None, **_):
if variant == 'major_minor_nodot':
return GRAAL_VERSION_MAJ_MIN.replace(".", "")
elif variant == 'major_minor':
return GRAAL_VERSION_MAJ_MIN
elif variant == 'binary':
# PythonLanguage and PythonResource consume this data, and they assume 3 components, so we cap the list size
# to 3 although the version may have even more components
return "".join([chr(int(p) + ord(VERSION_BASE)) for p in GRAAL_VERSION.split(".")[:3]])
elif variant == 'hex':
parts = GRAAL_VERSION.split(".")
num = 0
for i in range(3):
num