diff --git a/ci.jsonnet b/ci.jsonnet index 19f1d5054d..344e1f6984 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -5,7 +5,7 @@ (import "ci/python-gate.libsonnet") + (import "ci/python-bench.libsonnet") + { - overlay: "a8df46e16d6fcae69e9a7c42c632131fdf6e043e", + overlay: "907f84a98fe5736824d2964e8815d75d0511d39d", specVersion: "6", // Until buildbot issues around CI tiers are resolved, we cannot use them // tierConfig: self.tierConfig, @@ -208,7 +208,7 @@ "windows:amd64:jdk-latest" : tier3 + require(GRAAL_JDK_LATEST), }), "python-junit-bytecode-dsl": gpgate + platform_spec(no_jobs) + bytecode_dsl_gate("python-junit") + platform_spec({ - "linux:amd64:jdk21" : tier3 + require(GRAAL_JDK_LATEST), + "linux:amd64:jdk21" : daily + require(GRAAL_JDK_LATEST), }), "python-junit-maven": gpgate_maven + platform_spec(no_jobs) + platform_spec({ "linux:amd64:jdk21" : daily + t("00:30:00"), diff --git a/ci/graal/ci/ci_common/common-utils.libsonnet b/ci/graal/ci/ci_common/common-utils.libsonnet index 794af17efc..a476dc8d31 100644 --- a/ci/graal/ci/ci_common/common-utils.libsonnet +++ b/ci/graal/ci/ci_common/common-utils.libsonnet @@ -25,7 +25,7 @@ # Adds a CI build predicate to `build` if it is a gate such that it is only # run if a top level CI file or a non-documentation file in any of `suites` has been updated add_gate_predicate(build, suites, extra_includes=[], extra_excludes=[]):: - if std.member(build.targets, "gate") || std.member(build.targets, "tier1") || std.member(build.targets, "tier2") || std.member(build.targets, "tier3") then + if std.member(build.targets, "gate") then build + { guard+: { includes+: [ suite + "/**" for suite in suites ] + extra_includes + $.top_level_ci, diff --git a/ci/graal/ci/ci_common/musl-common.libsonnet b/ci/graal/ci/ci_common/musl-common.libsonnet index b9a788fac6..3bae934d67 100644 --- a/ci/graal/ci/ci_common/musl-common.libsonnet +++ b/ci/graal/ci/ci_common/musl-common.libsonnet @@ -3,8 +3,8 @@ musl_dependency:: { downloads+: { MUSL_TOOLCHAIN: { - name: 'toolchain-gcc-10.2.1-musl', - version: '1.2.2', + name: 'toolchain-gcc-10.3.0-zlib-1.2.13-musl', + version: '1.2.5.1', platformspecific: true, }, }, diff --git a/ci/graal/ci/common.jsonnet b/ci/graal/ci/common.jsonnet index 4272ca2d15..74d4ac1dc9 100644 --- a/ci/graal/ci/common.jsonnet +++ b/ci/graal/ci/common.jsonnet @@ -57,9 +57,6 @@ local common_json = import "../common.json"; for name in ["oraclejdk21"] + variants("labsjdk-ce-21") + variants("labsjdk-ee-21") } + { 'oraclejdk24': jdk_base + common_json.jdks["oraclejdk24"] + { jdk_version:: 24 }, - } + { - [name]: jdk_base + common_json.jdks[name] + { jdk_version:: 25 } - for name in ["oraclejdk25"] + variants("labsjdk-ce-25") + variants("labsjdk-ee-25") } + { [name]: jdk_base + common_json.jdks[name] + { jdk_version:: parse_labsjdk_version(self), jdk_name:: "jdk-latest"} for name in ["oraclejdk-latest"] + variants("labsjdk-ce-latest") + variants("labsjdk-ee-latest") @@ -145,11 +142,6 @@ local common_json = import "../common.json"; common_catch_files: { catch_files+: [ - # There are additional catch_files-like patterns in buildbot/graal/catcher.py for: - # * hs_err_pid*.log files - # * Dumping IGV graphs to (?P.+(\.gv\.xml|\.bgv)) - # * CFGPrinter: Output to file (?P.*compilations-.+\.cfg) - # There are defined there for efficiency reasons. # Keep in sync with jdk.graal.compiler.debug.StandardPathUtilitiesProvider#DIAGNOSTIC_OUTPUT_DIRECTORY_MESSAGE_REGEXP "Graal diagnostic output saved in '(?P[^']+)'", # Keep in sync with jdk.graal.compiler.debug.DebugContext#DUMP_FILE_MESSAGE_REGEXP @@ -225,19 +217,15 @@ local common_json = import "../common.json"; } }, - # ProGuard does not yet run on JDK 25 - proguard: { - downloads+: if 'jdk_version' in self && self.jdk_version > 21 then { - TOOLS_JAVA_HOME: jdks_data['oraclejdk24'], - IGV_JAVA_HOME: jdks_data['oraclejdk21'], - } else {}, - }, - # GR-49566: SpotBugs does not yet run on JDK 22 - spotbugs: { + local code_tools = { downloads+: if 'jdk_version' in self && self.jdk_version > 21 then { TOOLS_JAVA_HOME: jdks_data['oraclejdk21'], } else {}, }, + # GR-46676: ProGuard does not yet run on JDK 22 + proguard: code_tools, + # GR-49566: SpotBugs does not yet run on JDK 22 + spotbugs: code_tools, sulong:: self.cmake + { packages+: if self.os == "windows" then { @@ -277,10 +265,9 @@ local common_json = import "../common.json"; graalpy:: self.gradle + self.cmake + { packages+: if (self.os == "linux") then { - libffi: '==3.2.1', - bzip2: '==1.0.6', - zlib: '==1.2.11', - maven: "==3.6.3", + libffi: '>=3.2.1', + bzip2: '>=1.0.6', + maven: ">=3.3.9", } else {}, }, @@ -366,15 +353,6 @@ local common_json = import "../common.json"; # Job frequencies # *************** frequencies: { - tier1: { - targets+: ["tier1"], - }, - tier2: { - targets+: ["tier2"], - }, - tier3: { - targets+: ["tier3"], - }, gate: { targets+: ["gate"], }, diff --git a/ci/graal/common.json b/ci/graal/common.json index 5b3c75de13..86cb722ac2 100644 --- a/ci/graal/common.json +++ b/ci/graal/common.json @@ -4,11 +4,11 @@ "Jsonnet files should not include this file directly but use ci/common.jsonnet instead." ], - "mx_version": "7.58.9", + "mx_version": "7.54.6.1", "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet", "jdks": { - "galahad-jdk": {"name": "jpg-jdk", "version": "25", "build_id": "jdk-26+6-570", "platformspecific": true, "extrabundles": ["static-libs"]}, + "galahad-jdk": {"name": "jpg-jdk", "version": "25", "build_id": "jdk-25+26-3156", "platformspecific": true, "extrabundles": ["static-libs"]}, "oraclejdk17": {"name": "jpg-jdk", "version": "17.0.7", "build_id": "jdk-17.0.7+8", "platformspecific": true, "extrabundles": ["static-libs"]}, "labsjdk-ce-17": {"name": "labsjdk", "version": "ce-17.0.7+4-jvmci-23.1-b02", "platformspecific": true }, @@ -45,21 +45,13 @@ "oraclejdk24": {"name": "jpg-jdk", "version": "24", "build_id": "jdk-24.0.1+9", "platformspecific": true, "extrabundles": ["static-libs"]}, - "oraclejdk25": {"name": "jpg-jdk", "version": "25", "build_id": "jdk-25+26", "platformspecific": true, "extrabundles": ["static-libs"]}, - "labsjdk-ce-25": {"name": "labsjdk", "version": "ce-25+26-jvmci-b01", "platformspecific": true }, - "labsjdk-ce-25Debug": {"name": "labsjdk", "version": "ce-25+26-jvmci-b01-debug", "platformspecific": true }, - "labsjdk-ce-25-llvm": {"name": "labsjdk", "version": "ce-25+26-jvmci-b01-sulong", "platformspecific": true }, - "labsjdk-ee-25": {"name": "labsjdk", "version": "ee-25+26-jvmci-b01", "platformspecific": true }, - "labsjdk-ee-25Debug": {"name": "labsjdk", "version": "ee-25+26-jvmci-b01-debug", "platformspecific": true }, - "labsjdk-ee-25-llvm": {"name": "labsjdk", "version": "ee-25+26-jvmci-b01-sulong", "platformspecific": true }, - - "oraclejdk-latest": {"name": "jpg-jdk", "version": "26", "build_id": "jdk-26+6", "platformspecific": true, "extrabundles": ["static-libs"]}, - "labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-26+6-jvmci-b01", "platformspecific": true }, - "labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-26+6-jvmci-b01-debug", "platformspecific": true }, - "labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-26+5-jvmci-b01-sulong", "platformspecific": true }, - "labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-26+6-jvmci-b01", "platformspecific": true }, - "labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-26+6-jvmci-b01-debug", "platformspecific": true }, - "labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-26+5-jvmci-b01-sulong", "platformspecific": true } + "oraclejdk-latest": {"name": "jpg-jdk", "version": "25", "build_id": "jdk-25+37", "platformspecific": true, "extrabundles": ["static-libs"]}, + "labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-25+37-jvmci-b01", "platformspecific": true }, + "labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-25+37-jvmci-b01-debug", "platformspecific": true }, + "labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-25+37-jvmci-b01-sulong", "platformspecific": true }, + "labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-25+37-jvmci-b01", "platformspecific": true }, + "labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-25+37-jvmci-b01-debug", "platformspecific": true }, + "labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-25+37-jvmci-b01-sulong", "platformspecific": true } }, "eclipse": { diff --git a/ci/python-gate.libsonnet b/ci/python-gate.libsonnet index dbe350b221..a024a8b933 100644 --- a/ci/python-gate.libsonnet +++ b/ci/python-gate.libsonnet @@ -262,7 +262,7 @@ tier1:: $.target("tier1"), tier2:: $.target("tier2"), tier3:: $.target("tier3"), - post_merge:: $.target("tier4") + task_spec({name_target:: "post_merge"}), + post_merge:: $.target("post-merge") + task_spec({name_target:: "post_merge"}), bench:: $.target(graal_common.frequencies.bench.targets[0]), on_demand:: $.target(graal_common.frequencies.on_demand.targets[0]) + task_spec({name_target:: "on_demand"}), diff --git a/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java b/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java index 781ec18a3b..3f937cbc8c 100644 --- a/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java +++ b/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java @@ -182,6 +182,7 @@ protected List preprocessArguments(List givenArgs, Map(); boolean posixBackendSpecified = false; boolean sha3BackendSpecified = false; + boolean compressionBackendSpecified = false; boolean installSignalHandlersSpecified = false; boolean isolateNativeModulesSpecified = false; for (Iterator argumentIterator = arguments.iterator(); argumentIterator.hasNext();) { @@ -271,7 +272,8 @@ protected List preprocessArguments(List givenArgs, Map preprocessArguments(List givenArgs, Map preprocessArguments(List givenArgs, Map17 17 UTF-8 - 26.0.0 + 25.0.1 diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_capi.py b/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_capi.py index 0c53ab7f6c..b44a86890d 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_capi.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_capi.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -54,7 +54,7 @@ def check_autopatched(source, expected): autopatch_capi.auto_patch(f.name, False) f.seek(0) actual = f.read() - assert actual == expected, f"Autopatch didn't make expected changes. Expected:\n{expected}\nActual:{actual}" + assert actual == expected, f"Autopatch didn't make expected changes. Expected:\n{expected}\nActual:\n{actual}" def test_replace_field_access(): @@ -106,3 +106,29 @@ def test_replace_field_access(): #endif ''', ) + check_autopatched( + ''' + #if SOME_MACRO + ((PyCFunctionObject *) func)->m_ml, + #else + ((PyCFunctionObject *) func)->m_ml, + #endif + ''', + ''' + #if SOME_MACRO + GraalPyCFunction_GetMethodDef((PyObject*)(((PyCFunctionObject *) func))), + #else + GraalPyCFunction_GetMethodDef((PyObject*)(((PyCFunctionObject *) func))), + #endif + ''', + ) + check_autopatched( + ''' + // PyList_SET_ITEM(). + L->ob_item[len] = x; + ''', + ''' + // PyList_SET_ITEM(). + PySequence_Fast_ITEMS((PyObject*)L)[len] = x; + ''', + ) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_posix.py b/graalpython/com.oracle.graal.python.test/src/tests/test_posix.py index 247f11b72e..0fd52da639 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_posix.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_posix.py @@ -173,6 +173,31 @@ def test_pipe(self): os.close(fd1) os.close(fd2) + @unittest.skipIf(sys.platform != 'linux', 'mkfifo is a linux command') + def test_seek_pipe(self): + new_file_path = './myscript.sh' + with io.open(new_file_path, 'w') as script: + script.write("""#!/bin/sh + mkfifo testpipe + echo "4" > testpipe & + """) + try: + st = os.stat(new_file_path) + os.chmod(new_file_path, st.st_mode | stat.S_IEXEC) + os.system(new_file_path) + with io.open("testpipe", "rb") as r: + out = r.read(1) + assert out == b"4", out + finally: + try: + os.remove(new_file_path) + except: + pass + try: + os.remove("testpipe") + except: + pass + def test_mkdir_rmdir(self): os.mkdir(TEST_FULL_PATH1) try: diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py b/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py index dbfd881928..672468b57f 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py @@ -162,6 +162,21 @@ def test_java_asserts(self): result = subprocess.run([sys.executable, "-c", "import __graalpython__; not __graalpython__.java_assert()"]) assert result.returncode == 0 + def test_subprocess_inherits_environ(self): + import os + import subprocess + prev = os.environ.get("FOOBAR") + try: + expected_value = f"42{prev}".strip() + os.environ["FOOBAR"] = expected_value + out = subprocess.check_output([sys.executable, '-c', "import os; print(os.environ['FOOBAR'])"]).decode().strip() + assert out == expected_value, f"{out!r} != {expected_value!r}" + finally: + if prev: + os.environ["FOOBAR"] = prev + else: + del os.environ["FOOBAR"] + @unittest.skipUnless(sys.implementation.name == 'graalpy', "GraalPy-specific test") @unittest.skipIf(sys.platform == 'win32', "TODO the cmd replacement breaks the test") def test_graal_python_args(self): diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_zlib.py b/graalpython/com.oracle.graal.python.test/src/tests/test_zlib.py index 8dd5aacc79..de9fd69d4b 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_zlib.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_zlib.py @@ -14,11 +14,12 @@ def _disable_native_zlib(self, flag): return None __graalpython__ = GP() +import binascii import os +import random +import sys import unittest import zlib -import binascii -import sys pintNumber = 98765432109876543210 longNumber = 9876543210 @@ -271,3 +272,35 @@ def test_GR65704(): __graalpython__._disable_native_zlib(False) assert decompressed == contents + +def test_large_chunk(): + contents = random.randbytes(5000) + wbits = 31 + + __graalpython__._disable_native_zlib(True) + + compressed = zlib.compress(contents, wbits=wbits) + decompressor = zlib.decompressobj(wbits=wbits) + + decompressed = decompressor.decompress(compressed) + + __graalpython__._disable_native_zlib(False) + + assert decompressed == contents + +def test_various_chunks(): + contents = random.randbytes(5000) + wbits = 31 + + __graalpython__._disable_native_zlib(True) + + compressed = zlib.compress(contents, wbits=wbits) + decompressor = zlib.decompressobj(wbits=wbits) + + decompressed = decompressor.decompress(compressed[:10]) + decompressed += decompressor.decompress(compressed[10:200]) + decompressed += decompressor.decompress(compressed[200:]) + + __graalpython__._disable_native_zlib(False) + + assert decompressed == contents diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/zlib/JavaDecompress.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/zlib/JavaDecompress.java index ade0a03d6f..b9752be544 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/zlib/JavaDecompress.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/zlib/JavaDecompress.java @@ -47,6 +47,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.util.zip.DataFormatException; @@ -112,7 +113,7 @@ public Inflater getInflater() { return inf; } - public void setInput() throws IOException { + public void fillInput() throws IOException { fill(); } } @@ -162,7 +163,7 @@ private static boolean isGZIPStreamReady(DecompressStream stream, byte[] data, i // GZIPInputStream will read the header during initialization stream.stream = new GZIPDecompressStream(stream.in); stream.inflater = stream.stream.getInflater(); - stream.stream.setInput(); + stream.stream.fillInput(); return true; } } catch (ZipException ze) { @@ -178,7 +179,7 @@ private static boolean isGZIPStreamFinishing(DecompressStream stream, byte[] dat stream.in.append(data, 0, length); try { if (stream.in.length() >= HEADER_TRAILER_SIZE) { - stream.stream.setInput(); + stream.stream.fillInput(); // this should trigger reading trailer stream.stream.read(); stream.stream = null; @@ -243,13 +244,25 @@ private byte[] createByteArray(byte[] bytes, int length, int maxLength, int bufS return EMPTY_BYTE_ARRAY; } - int maxLen = maxLength == 0 ? Integer.MAX_VALUE : maxLength; + int maxLen = maxLength <= 0 ? Integer.MAX_VALUE : maxLength; byte[] result = new byte[Math.min(maxLen, bufSize)]; - int bytesWritten = result.length; ByteArrayOutputStream baos = new ByteArrayOutputStream(); boolean zdictIsSet = false; - while (baos.size() < maxLen && bytesWritten == result.length) { + while (baos.size() < maxLen && !stream.inflater.finished()) { + if (stream.inflater.needsInput()) { + if (stream.stream == null) { + break; + } + try { + stream.stream.fillInput(); + } catch (EOFException e) { + break; + } catch (IOException e) { + throw CompilerDirectives.shouldNotReachHere(e); + } + } + int bytesWritten; try { int len = Math.min(maxLen - baos.size(), result.length); bytesWritten = stream.inflater.inflate(result, 0, len); @@ -257,8 +270,7 @@ private byte[] createByteArray(byte[] bytes, int length, int maxLength, int bufS if (getZdict().length > 0) { setDictionary(); zdictIsSet = true; - // we inflate again with a dictionary - bytesWritten = stream.inflater.inflate(result, 0, len); + continue; } else { throw PRaiseNode.raiseStatic(nodeForRaise, ZLibError, WHILE_SETTING_ZDICT); } @@ -320,7 +332,7 @@ protected static byte[] decompress(byte[] bytes, int length, int wbits, int bufs private void saveUnconsumedInput(byte[] data, int length, byte[] unusedDataBytes, int unconsumedTailLen, Node inliningTarget) { int unusedLen = getRemaining(); - byte[] tail = PythonUtils.arrayCopyOfRange(data, length - unusedLen, length); + byte[] tail = PythonUtils.arrayCopyOfRange(data, Math.max(0, length - unusedLen), length); PythonLanguage language = PythonLanguage.get(inliningTarget); if (isEof()) { if (unconsumedTailLen > 0) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/exception/OSErrorEnum.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/exception/OSErrorEnum.java index fc0f608b1e..135d39bcd9 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/exception/OSErrorEnum.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/exception/OSErrorEnum.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -312,7 +312,7 @@ public static ErrorAndMessagePair fromException(Exception e, TruffleString.Equal return new ErrorAndMessagePair(oserror, oserror.getMessage()); } } else { // Generic IOException - OSErrorEnum oserror = tryFindErrnoFromMessage(e); + OSErrorEnum oserror = tryFindErrnoFromMessage(e, eqNode); if (oserror == null) { return new ErrorAndMessagePair(OSErrorEnum.EIO, getMessage(e)); } else { @@ -360,15 +360,13 @@ private static TruffleString getReason(FileSystemException e) { } @TruffleBoundary - private static OSErrorEnum tryFindErrnoFromMessage(Exception e) { - if (e.getMessage().contains("Broken pipe")) { - return OSErrorEnum.EPIPE; - } - Matcher m = ERRNO_PATTERN.matcher(e.getMessage()); + private static OSErrorEnum tryFindErrnoFromMessage(Exception e, TruffleString.EqualNode eqNode) { + String message = e.getMessage(); + Matcher m = ERRNO_PATTERN.matcher(message); if (m.find()) { return fromNumber(Integer.parseInt(m.group(1))); } - return null; + return OSErrorEnum.fromMessage(toTruffleStringUncached(message), eqNode); } @ValueType diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java index 75773e27b0..9198f7c151 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java @@ -527,10 +527,15 @@ public Buffer read(int fd, long length, } @TruffleBoundary - private static Buffer readBytesFromChannel(ReadableByteChannel channel, long size) throws IOException { + private static Buffer readBytesFromChannel(ReadableByteChannel channel, long sizeIn) throws IOException { + long size = sizeIn; if (channel instanceof SeekableByteChannel seekableByteChannel) { - long availableSize = seekableByteChannel.size() - seekableByteChannel.position(); - size = Math.min(size, availableSize); + try { + long availableSize = seekableByteChannel.size() - seekableByteChannel.position(); + size = Math.min(size, availableSize); + } catch (IOException e) { + // pass and read what we can + } } size = Math.min(size, MAX_READ); ByteBuffer dst = ByteBuffer.allocate((int) size); @@ -2310,6 +2315,8 @@ public int forkExec(Object[] executables, Object[] args, Object cwd, Object[] en throw createUnsupportedFeature("Only key=value environment variables are supported in fork_exec"); } } + } else { + envMap = new HashMap<>(environ); } String[] argStrings; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIBz2Support.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIBz2Support.java index acb8b48353..70bcd84756 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIBz2Support.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIBz2Support.java @@ -153,7 +153,7 @@ public String signature() { @CompilerDirectives.CompilationFinal private boolean available; private NFIBz2Support(PythonContext context, NativeLibrary.NFIBackend backend, String noNativeAccessHelp) { - if (context.isNativeAccessAllowed()) { + if (context.useNativeCompressionModules()) { this.pythonContext = context; this.typedNativeLib = NativeLibrary.create(PythonContext.getSupportLibName(SUPPORTING_NATIVE_LIB_NAME), Bz2NativeFunctions.values(), backend, noNativeAccessHelp, false); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFILZMASupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFILZMASupport.java index 4155cfee22..c0595c9639 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFILZMASupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFILZMASupport.java @@ -293,7 +293,7 @@ public String signature() { @CompilerDirectives.CompilationFinal private boolean available; private NFILZMASupport(PythonContext context, NativeLibrary.NFIBackend backend, String noNativeAccessHelp) { - if (context.isNativeAccessAllowed()) { + if (context.useNativeCompressionModules()) { this.pythonContext = context; this.typedNativeLib = NativeLibrary.create(PythonContext.getSupportLibName(SUPPORTING_NATIVE_LIB_NAME), LZMANativeFunctions.values(), backend, noNativeAccessHelp, true); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIZlibSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIZlibSupport.java index 8043d34878..97728360fb 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIZlibSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIZlibSupport.java @@ -269,7 +269,7 @@ public String signature() { @CompilerDirectives.CompilationFinal private boolean available; private NFIZlibSupport(PythonContext context, NativeLibrary.NFIBackend backend, String noNativeAccessHelp) { - if (context.isNativeAccessAllowed()) { + if (context.useNativeCompressionModules()) { this.pythonContext = context; this.typedNativeLib = NativeLibrary.create(PythonContext.getSupportLibName(SUPPORTING_NATIVE_LIB_NAME), ZlibNativeFunctions.values(), backend, noNativeAccessHelp, true); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java index 5e93534742..975bad40be 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java @@ -2335,6 +2335,15 @@ public boolean isExecutableAccessAllowed() { return getEnv().isHostLookupAllowed() || isNativeAccessAllowed(); } + public boolean useNativeCompressionModules() { + if (isNativeAccessAllowed()) { + TruffleString option = getLanguage().getEngineOption(PythonOptions.CompressionModulesBackend); + TruffleString.EqualNode eqNode = TruffleString.EqualNode.getUncached(); + return !eqNode.execute(T_JAVA, option, TS_ENCODING); + } + return false; + } + /** * Trigger any pending asynchronous actions */ diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java index 00adc12ad0..58e6f245b6 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java @@ -195,12 +195,15 @@ public static void checkBytecodeDSLEnv() { } })); - @EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the POSIX module.", usageSyntax = "java|native|llvm", stability = OptionStability.STABLE) // + @EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the POSIX module.", usageSyntax = "java|native", stability = OptionStability.STABLE) // public static final OptionKey PosixModuleBackend = new OptionKey<>(T_JAVA, TS_OPTION_TYPE); @EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the Sha3 module.", usageSyntax = "java|native", stability = OptionStability.STABLE) // public static final OptionKey Sha3ModuleBackend = new OptionKey<>(T_JAVA, TS_OPTION_TYPE); + @EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the Zlib, Bz2, and LZMA modules.", usageSyntax = "java|native", stability = OptionStability.STABLE) // + public static final OptionKey CompressionModulesBackend = new OptionKey<>(T_JAVA, TS_OPTION_TYPE); + @Option(category = OptionCategory.USER, help = "Install default signal handlers on startup", usageSyntax = "true|false", stability = OptionStability.STABLE) // public static final OptionKey InstallSignalHandlers = new OptionKey<>(false); diff --git a/graalpython/graalpy-jbang/examples/hello.java b/graalpython/graalpy-jbang/examples/hello.java index 1cd6805367..b545258e3c 100644 --- a/graalpython/graalpy-jbang/examples/hello.java +++ b/graalpython/graalpy-jbang/examples/hello.java @@ -40,7 +40,7 @@ */ ///usr/bin/env jbang "$0" "$@" ; exit $? //JAVA 17+ -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:26.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.1} // specify python packages and their versions as if used with pip //PIP termcolor==2.2 diff --git a/graalpython/graalpy-jbang/templates/graalpy-template.java.qute b/graalpython/graalpy-jbang/templates/graalpy-template.java.qute index 1c397a3f35..1181dc2492 100644 --- a/graalpython/graalpy-jbang/templates/graalpy-template.java.qute +++ b/graalpython/graalpy-jbang/templates/graalpy-template.java.qute @@ -5,7 +5,7 @@ {/for} {#if dependencies.isEmpty()}// //DEPS {/if} {| -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:26.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.1} // specify python packages and their versions as if used with pip //PIP termcolor==2.2 |} diff --git a/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute b/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute index 094182f80c..49f88bbbd5 100644 --- a/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute +++ b/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute @@ -8,7 +8,7 @@ //REPOS mc=https://repo1.maven.org/maven2/ //REPOS local=file://{path_to_local_repo} {| -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:26.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.1} // specify python packages and their versions as if used with pip //PIP termcolor==2.2 |} diff --git a/graalpython/lib-graalpython/modules/autopatch_capi.py b/graalpython/lib-graalpython/modules/autopatch_capi.py index 1551aa218c..49382c5fc6 100644 --- a/graalpython/lib-graalpython/modules/autopatch_capi.py +++ b/graalpython/lib-graalpython/modules/autopatch_capi.py @@ -65,8 +65,13 @@ def replace_field_access(contents, match, replacement, assignment): start, end = match.span(1) level = 0 - def consume_whitespace_backwards(idx): - while idx >= 0 and contents[idx].isspace(): + def consume_whitespace_backwards(idx, newlines=True): + while idx >= 0 and contents[idx].isspace() and (not newlines or contents[idx] != '\n'): + idx -= 1 + return idx + + def consume_line_backwards(idx): + while idx >= 0 and contents[idx] != '\n': idx -= 1 return idx @@ -124,6 +129,13 @@ def consume_identifier_backwards(idx): else: idx += 1 break + idx = consume_whitespace_backwards(idx, newlines=False) + if contents[idx + 1] == '\n': + # Get previous line. If it's a comment or a preprocessor directive, stop + line_start = consume_whitespace_forward(consume_line_backwards(idx)) + if contents[line_start: line_start + 2].startswith(('#', '//')): + idx += 1 + break idx = consume_whitespace_backwards(idx) receiver_start = consume_whitespace_forward(idx) diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py index 7c76ef12b2..753b569ce2 100644 --- a/mx.graalpython/mx_graalpython.py +++ b/mx.graalpython/mx_graalpython.py @@ -1925,6 +1925,7 @@ def bytecode_dsl_build_args(): '--enable-native-access=org.graalvm.shadowed.jline', '-Dpolyglot.python.PosixModuleBackend=native', '-Dpolyglot.python.Sha3ModuleBackend=native', + '-Dpolyglot.python.CompressionModulesBackend=native', ] + bytecode_dsl_build_args(), language='python', default_vm_args=[ diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index 5c237ab663..f0237f8972 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -9,7 +9,7 @@ "name": "graalpython", "versionConflictResolution": "latest", - "version": "26.0.0", + "version": "25.0.1", "graalpython:pythonVersion": "3.12.8", "release": False, "groupId": "org.graalvm.python", @@ -53,7 +53,7 @@ }, { "name": "tools", - "version": "a180d4f836e7c10201e90c6fc4e06817008b1af7", + "version": "4391e7b150f764258d4a15ffb48b46a8c19ed71a", "subdir": True, "urls": [ {"url": "https://github.com/oracle/graal", "kind": "git"}, @@ -61,7 +61,7 @@ }, { "name": "regex", - "version": "a180d4f836e7c10201e90c6fc4e06817008b1af7", + "version": "4391e7b150f764258d4a15ffb48b46a8c19ed71a", "subdir": True, "urls": [ {"url": "https://github.com/oracle/graal", "kind": "git"}, @@ -810,6 +810,7 @@ '--enable-native-access=org.graalvm.shadowed.jline', "-Dpolyglot.python.PosixModuleBackend=native", "-Dpolyglot.python.Sha3ModuleBackend=native", + "-Dpolyglot.python.CompressionModulesBackend=native", ], "dynamicBuildArgs": "libpythonvm_build_args", },