From 1c429a67eab8b754f927151da725bfb97c460214 Mon Sep 17 00:00:00 2001 From: slozier Date: Thu, 24 Aug 2023 20:23:32 -0400 Subject: [PATCH 01/12] Speed up test_cgcheck (#1736) --- .../Cases/IronPythonCasesManifest.ini | 3 -- Src/Scripts/generate.py | 38 +++++++++++++------ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini b/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini index e8166fb1b..214561311 100644 --- a/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini +++ b/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini @@ -184,9 +184,6 @@ RunCondition=NOT $(IS_OSX) # ctypes tests not prepared for macOS [IronPython.scripts.test_builder] Ignore=true -[IronPython.scripts.test_cgcheck] -Timeout=600000 # 10 minute timeout - [IronPython.scripts.test_parrot] Ignore=true diff --git a/Src/Scripts/generate.py b/Src/Scripts/generate.py index c0e2979df..6b7ac5b2f 100755 --- a/Src/Scripts/generate.py +++ b/Src/Scripts/generate.py @@ -2,6 +2,7 @@ # The .NET Foundation licenses this file to you under the Apache 2.0 License. # See the LICENSE file in the project root for more information. +import functools import re import sys import os @@ -23,7 +24,8 @@ def get_root_dir(): os.path.join(root_dir, "Src", "StdLib"), ] -START = "#region Generated %s" +START_COMMON = "#region Generated" +START = START_COMMON + " %s" END = "#endregion" PREFIX = r"^([ \t]*)" @@ -131,6 +133,27 @@ def text(self): def conditions(self): return ConditionWriter(self) +@functools.lru_cache() +def find_candidates(dirname): + def listdir(dirname): + if dirname in exclude_directories: + return + + for file in os.listdir(dirname): + if file == "obj": continue # obj folders are not interesting... + filename = os.path.join(dirname, file) + if os.path.isdir(filename): + yield from listdir(filename) + elif filename.endswith(".cs") and not file == "StandardTestStrings.cs": # TODO: fix encoding of StandardTestStrings.cs + yield filename + + res = [] + for file in listdir(dirname): + with open(file, encoding='latin-1') as f: + if START_COMMON in f.read(): + res.append(file) + return res + class CodeGenerator: def __init__(self, name, generator): self.generator = generator @@ -151,19 +174,10 @@ def do_generate(self): result.append(g.generate()) return result - def do_dir(self, dirname): - if dirname in exclude_directories: - return - for file in os.listdir(dirname): - filename = os.path.join(dirname, file) - if os.path.isdir(filename): - self.do_dir(filename) - elif filename.endswith(".cs") and not file == "StandardTestStrings.cs": # TODO: fix encoding of StandardTestStrings.cs - self.do_file(filename) - def doit(self): for src_dir in source_directories: - self.do_dir(src_dir) + for file in find_candidates(src_dir): + self.do_file(file) for g in self.generators: g.collect_info() return self.do_generate() From 63a881915d8bd2b3e2e0351079fc9bd4c97ad593 Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Sat, 26 Aug 2023 06:40:46 -0700 Subject: [PATCH 02/12] Performance improvements of reading and writing through StreamBox (#1732) * Performance improvements of reading and writing through StreamBox * Optimize StreamBox.ReadInto --- Src/IronPython.Modules/nt.cs | 5 +-- Src/IronPython/Modules/_fileio.cs | 12 +----- Src/IronPython/Runtime/BufferProtocol.cs | 45 ++++++++++++++++++--- Src/IronPython/Runtime/PythonFileManager.cs | 33 +++++++++++++-- 4 files changed, 73 insertions(+), 22 deletions(-) diff --git a/Src/IronPython.Modules/nt.cs b/Src/IronPython.Modules/nt.cs index 2e03affcf..f6f920a3c 100644 --- a/Src/IronPython.Modules/nt.cs +++ b/Src/IronPython.Modules/nt.cs @@ -1802,13 +1802,12 @@ public static PythonTuple waitpid(int pid, int options) { public static int write(CodeContext/*!*/ context, int fd, [NotNone] IBufferProtocol data) { try { + using var buffer = data.GetBuffer(); PythonContext pythonContext = context.LanguageContext; var streams = pythonContext.FileManager.GetStreams(fd); - using var buffer = data.GetBuffer(); - var bytes = buffer.AsReadOnlySpan(); if (!streams.WriteStream.CanWrite) throw PythonOps.OSError(9, "Bad file descriptor"); - return streams.Write(bytes); + return streams.Write(buffer); } catch (Exception e) { throw ToPythonException(e); } diff --git a/Src/IronPython/Modules/_fileio.cs b/Src/IronPython/Modules/_fileio.cs index 5d90626e0..1069ea15b 100644 --- a/Src/IronPython/Modules/_fileio.cs +++ b/Src/IronPython/Modules/_fileio.cs @@ -351,14 +351,7 @@ public BigInteger readinto([NotNone] IBufferProtocol buffer) { _checkClosed(); - var span = pythonBuffer.AsSpan(); - for (int i = 0; i < span.Length; i++) { - int b = _streams.ReadStream.ReadByte(); - if (b == -1) return i; - span[i] = (byte)b; - } - - return span.Length; + return _streams.ReadInto(pythonBuffer); } public override BigInteger readinto(CodeContext/*!*/ context, object buf) { @@ -444,10 +437,9 @@ public override bool writable(CodeContext/*!*/ context) { public override BigInteger write(CodeContext/*!*/ context, object b) { var bufferProtocol = Converter.Convert(b); using var buffer = bufferProtocol.GetBuffer(); - var bytes = buffer.AsReadOnlySpan(); EnsureWritable(); - return _streams.Write(bytes); + return _streams.Write(buffer); } #endregion diff --git a/Src/IronPython/Runtime/BufferProtocol.cs b/Src/IronPython/Runtime/BufferProtocol.cs index 424055402..f537ae709 100644 --- a/Src/IronPython/Runtime/BufferProtocol.cs +++ b/Src/IronPython/Runtime/BufferProtocol.cs @@ -308,23 +308,56 @@ public static void CopyTo(this IPythonBuffer buffer, Span dest) { } } + /// + /// Obtain the underlying array, if possible. + /// The returned array is unsafe because it should not be written to. + /// internal static byte[]? AsUnsafeArray(this IPythonBuffer buffer) { if (!buffer.IsCContiguous()) return null; - if (buffer.Object is Bytes b) - return b.UnsafeByteArray; - - if (buffer.Object is Memory mem) { - if (MemoryMarshal.TryGetArray(mem, out ArraySegment seg) && seg.Array != null && seg.Offset == 0 && seg.Count == seg.Array.Length) + ReadOnlySpan bufdata = buffer.AsReadOnlySpan(); + if (buffer.Object is Bytes b) { + if (b.UnsafeByteArray.AsSpan() == bufdata) + return b.UnsafeByteArray; + } else if (buffer.Object is ByteArray ba) { + byte[] arrdata = ba.UnsafeByteList.Data; + if (arrdata.AsSpan() == bufdata) + return arrdata; + } else if (buffer.Object is Memory mem) { + if (MemoryMarshal.TryGetArray(mem, out ArraySegment seg) && seg.Array is not null && seg.Array.AsSpan() == bufdata) return seg.Array; } else if (buffer.Object is ReadOnlyMemory rom) { - if (MemoryMarshal.TryGetArray(rom, out ArraySegment seg) && seg.Array != null && seg.Offset == 0 && seg.Count == seg.Array.Length) + if (MemoryMarshal.TryGetArray(rom, out ArraySegment seg) && seg.Array is not null && seg.Array.AsSpan() == bufdata) + return seg.Array; + } + + return null; + } + + /// + /// Obtain the underlying writable array, if possible. + /// The returned array is unsafe because it can be longer than the buffer. + /// + internal static byte[]? AsUnsafeWritableArray(this IPythonBuffer buffer) { + if (!buffer.IsCContiguous() || buffer.IsReadOnly) + return null; + + Span bufdata = buffer.AsSpan(); + if (buffer.Object is ByteArray ba) { + byte[] arrdata = ba.UnsafeByteList.Data; + if (UseSameMemory(arrdata, bufdata)) + return arrdata; + } else if (buffer.Object is Memory mem) { + if (MemoryMarshal.TryGetArray(mem, out ArraySegment seg) && seg.Array is not null && UseSameMemory(seg.Array, bufdata)) return seg.Array; } return null; } + + private static bool UseSameMemory(byte[] arr, ReadOnlySpan span) + => arr.Length >= span.Length && arr.AsSpan(0, span.Length) == span; } public ref struct BufferBytesEnumerator { diff --git a/Src/IronPython/Runtime/PythonFileManager.cs b/Src/IronPython/Runtime/PythonFileManager.cs index f5c3c9aa8..18144576b 100644 --- a/Src/IronPython/Runtime/PythonFileManager.cs +++ b/Src/IronPython/Runtime/PythonFileManager.cs @@ -109,17 +109,44 @@ public byte[] Read(int count) { return buffer; } - public int Write(ReadOnlySpan bytes) { + public int ReadInto(IPythonBuffer buffer) { #if NETCOREAPP + return _readStream.Read(buffer.AsSpan()); +#else + byte[]? bytes = buffer.AsUnsafeWritableArray(); + if (bytes is not null) { + return _readStream.Read(bytes, 0, buffer.NumBytes()); + } + + const int chunkSize = 0x400; // 1 KiB + bytes = new byte[chunkSize]; + var span = buffer.AsSpan(); + for (int pos = 0; pos < span.Length; pos += chunkSize) { + int toRead = Math.Min(chunkSize, span.Length - pos); + int hasRead = _readStream.Read(bytes, 0, toRead); + bytes.AsSpan(0, hasRead).CopyTo(span.Slice(pos)); + if (hasRead < toRead) return pos + hasRead; + } + return span.Length; +#endif + } + + public int Write(IPythonBuffer buffer) { + int count; +#if NETCOREAPP + ReadOnlySpan bytes = buffer.AsReadOnlySpan(); + count = bytes.Length; _writeStream.Write(bytes); #else - _writeStream.Write(bytes.ToArray(), 0, bytes.Length); + byte[] bytes = buffer.AsUnsafeArray() ?? buffer.AsUnsafeWritableArray() ?? buffer.ToArray(); + count = buffer.NumBytes(); + _writeStream.Write(bytes, 0, count); #endif _writeStream.Flush(); // IO at this level is not supposed to buffer so we need to call Flush. if (!IsSingleStream) { _readStream.Seek(_writeStream.Position, SeekOrigin.Begin); } - return bytes.Length; + return count; } public void Flush() { From dbcdb07cdf859a324a0c16526446d9a011284607 Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Sat, 26 Aug 2023 17:41:31 -0700 Subject: [PATCH 03/12] Use ArrayPool for buffered reading in StreamBox (#1738) --- Src/IronPython/Runtime/PythonFileManager.cs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Src/IronPython/Runtime/PythonFileManager.cs b/Src/IronPython/Runtime/PythonFileManager.cs index 18144576b..5070b9ffa 100644 --- a/Src/IronPython/Runtime/PythonFileManager.cs +++ b/Src/IronPython/Runtime/PythonFileManager.cs @@ -5,6 +5,7 @@ #nullable enable using System; +using System.Buffers; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; @@ -118,14 +119,18 @@ public int ReadInto(IPythonBuffer buffer) { return _readStream.Read(bytes, 0, buffer.NumBytes()); } - const int chunkSize = 0x400; // 1 KiB - bytes = new byte[chunkSize]; var span = buffer.AsSpan(); - for (int pos = 0; pos < span.Length; pos += chunkSize) { - int toRead = Math.Min(chunkSize, span.Length - pos); - int hasRead = _readStream.Read(bytes, 0, toRead); - bytes.AsSpan(0, hasRead).CopyTo(span.Slice(pos)); - if (hasRead < toRead) return pos + hasRead; + const int chunkSize = 0x1000; // 4 KiB, default buffer size of FileSteam + bytes = ArrayPool.Shared.Rent(chunkSize); + try { + for (int pos = 0; pos < span.Length; pos += chunkSize) { + int toRead = Math.Min(chunkSize, span.Length - pos); + int hasRead = _readStream.Read(bytes, 0, toRead); + bytes.AsSpan(0, hasRead).CopyTo(span.Slice(pos)); + if (hasRead < toRead) return pos + hasRead; + } + } finally { + ArrayPool.Shared.Return(bytes); } return span.Length; #endif From 42e6a0ecbf32f2e366854eda97e2f2007262383d Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Wed, 30 Aug 2023 06:14:10 -0700 Subject: [PATCH 04/12] Clarify version info requirements when submitting new issues (#1739) --- .github/ISSUE_TEMPLATE.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index fc054e3de..018a2c1ed 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,6 +1,6 @@ ### Prerequisites -The issue tracker is used to report bugs and request new features, NOT to ask questions. +The issue tracker is used to report bugs and request new features, **NOT** to ask questions. Questions should be posted in [Discussions](https://github.com/IronLanguages/ironpython3/discussions/categories/q-a) or to the users mailing list which can be accessed at https://ironpython.groups.io/g/users. @@ -19,10 +19,21 @@ https://ironpython.groups.io/g/users. 2. [Second Step] 3. [and so on...] -**Expected behavior:** [What you expected to happen] +**Expected behavior:** -**Actual behavior:** [What actually happened] +[What you expected to happen] -### Versions +**Actual behavior:** -You can get this information from executing `ipy -VV`. +[What actually happened] + +### Version Information + +If you are using the `ipy` console program, provide output of executing `ipy -VV`. If it is a local build, provide also the git hash of the commit used to build IronPython. Additionally, provide the type of the operating system used. + +If you are using the IronPython engine embedded in a .NET application, provide the version number of the NuGet package used (or if it is a local build, the git hash of the commit used to build IronPython), and the following info: + +* .NET platform used (choice from: .NET, .NET Core, .NET Framework, Mono, Unity), +* Version of the .NET platform used, +* Operating system used, +* Value of `sys.version`, from imported module `sys`. From 54081df1671a4eba26a7e6c4eb4593ba4423b3a9 Mon Sep 17 00:00:00 2001 From: slozier Date: Wed, 30 Aug 2023 09:14:46 -0400 Subject: [PATCH 05/12] Add clocks (#1737) * Add clocks * Disable test --- Src/IronPython.Modules/time.cs | 24 +++++++++++++++++++----- Tests/test_time_stdlib.py | 12 ++++++------ WhatsNewInPython33.md | 2 +- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/Src/IronPython.Modules/time.cs b/Src/IronPython.Modules/time.cs index fe3bf8312..f3aa4c69a 100644 --- a/Src/IronPython.Modules/time.cs +++ b/Src/IronPython.Modules/time.cs @@ -17,6 +17,7 @@ using System.Threading; using IronPython.Runtime; +using IronPython.Runtime.Exceptions; using IronPython.Runtime.Operations; using IronPython.Runtime.Types; @@ -106,18 +107,31 @@ public static double clock() { public static double perf_counter() => clock(); + public static double process_time() => Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds; + public static string ctime(CodeContext/*!*/ context) => asctime(context, localtime()); public static string ctime(CodeContext/*!*/ context, object? seconds) => asctime(context, localtime(seconds)); - public static object get_clock_info([NotNone] string name) { + public static object get_clock_info(CodeContext/*!*/ context, [NotNone] string name) { // TODO: Fill with correct values - if (name == "monotonic") - return new SimpleNamespace(new Dictionary { { "adjustable", false }, { "implementation", "Stopwatch.GetTimestamp" }, { "monotonic", true }, { "resolution", 0.015625 } }); - - throw new NotImplementedException(); + switch (name) { + case "monotonic": + return new SimpleNamespace(new Dictionary { { "adjustable", false }, { "implementation", "Stopwatch.GetTimestamp" }, { "monotonic", true }, { "resolution", 0.015625 } }); + case "perf_counter": + return new SimpleNamespace(new Dictionary { { "adjustable", false }, { "implementation", "Stopwatch.ElapsedTicks" }, { "monotonic", true }, { "resolution", 1e-7 } }); + case "process_time": + return new SimpleNamespace(new Dictionary { { "adjustable", false }, { "implementation", "Process.TotalProcessorTime" }, { "monotonic", true }, { "resolution", 1e-7 } }); + case "clock": + PythonOps.Warn(context, PythonExceptions.DeprecationWarning, "time.clock has been deprecated in Python 3.3 and will be removed from Python 3.8: use time.perf_counter or time.process_time instead"); + return new SimpleNamespace(new Dictionary { { "adjustable", false }, { "implementation", "Stopwatch.ElapsedTicks" }, { "monotonic", true }, { "resolution", 1e-7 } }); + case "time": + return new SimpleNamespace(new Dictionary { { "adjustable", true }, { "implementation", "DateTime.Now" }, { "monotonic", false }, { "resolution", 0.015625 } }); + default: + throw PythonOps.ValueError("unknown clock"); + } } public static void sleep(double tm) { diff --git a/Tests/test_time_stdlib.py b/Tests/test_time_stdlib.py index e3791ba4e..96357d893 100644 --- a/Tests/test_time_stdlib.py +++ b/Tests/test_time_stdlib.py @@ -6,7 +6,7 @@ ## Run selected tests from test_time from StdLib ## -from iptest import is_ironpython, generate_suite, run_test +from iptest import is_ironpython, is_osx, is_netcoreapp21, generate_suite, run_test import test.test_time @@ -14,22 +14,22 @@ def load_tests(loader, standard_tests, pattern): tests = loader.loadTestsFromModule(test.test_time) if is_ironpython: - failing_tests = [ test.test_time.TestAsctime4dyear('test_large_year'), # ValueError: year is too high test.test_time.TestAsctime4dyear('test_negative'), # ValueError: year is too low test.test_time.TimeTestCase('test_asctime'), # ValueError: year is too high test.test_time.TimeTestCase('test_asctime_bounding_check'), # ValueError: Hour, Minute, and Second parameters describe an un-representable DateTime. - test.test_time.TimeTestCase('test_clock'), # NotImplementedError: get_clock_info('clock') - test.test_time.TimeTestCase('test_get_clock_info'), # NotImplementedError: get_clock_info test.test_time.TimeTestCase('test_insane_timestamps'), # ValueError: unreasonable date/time test.test_time.TimeTestCase('test_mktime_error'), # ValueError: year is too low - test.test_time.TimeTestCase('test_process_time'), # AttributeError: 'module' object has no attribute 'process_time' test.test_time.TimeTestCase('test_strftime_bounding_check'), # ValueError: Hour, Minute, and Second parameters describe an un-representable DateTime. - test.test_time.TimeTestCase('test_time'), # NotImplementedError: get_clock_info('time') test.test_time.TimeTestCase('test_default_values_for_zero'), # AssertionError: '2000 01 01 00 00 00 1 001' != '2000 01 01 00 00 00 6 001' ] + if is_netcoreapp21 and is_osx: + failing_tests += [ + test.test_time.TimeTestCase('test_process_time'), # AssertionError + ] + skip_tests = [] return generate_suite(tests, failing_tests, skip_tests) diff --git a/WhatsNewInPython33.md b/WhatsNewInPython33.md index 7a76f99eb..08aee7fe7 100644 --- a/WhatsNewInPython33.md +++ b/WhatsNewInPython33.md @@ -117,7 +117,7 @@ Deprecated Python modules, functions and methods - [ ] `platform.popen()`: use the `subprocess` module. Check especially the "Replacing Older Functions with the `subprocess` Module" section (issue 11377). - [ ] The Windows `bytes` API has been deprecated in the `os` module. Use Unicode filenames, instead of `bytes` filenames, to not depend on the ANSI code page anymore and to support any filename. - [x] The `xml.etree.cElementTree` module is deprecated. The accelerator is used automatically whenever available. -- [ ] The behaviour of `time.clock()` depends on the platform: use the new `time.perf_counter()` or `time.process_time()` function instead, depending on your requirements, to have a well defined behaviour. +- [x] The behaviour of `time.clock()` depends on the platform: use the new `time.perf_counter()` or `time.process_time()` function instead, depending on your requirements, to have a well defined behaviour. - [x] ~~The `os.stat_float_times()` function is deprecated.~~ (Never implemented, but removed in Python 3.7) - `abc` module: + [ ] `abc.abstractproperty` has been deprecated, use `property` with `abc.abstractmethod()` instead. From e8ed79bd7f0f33eb2af1a538dd7e98767c86c211 Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Sat, 2 Sep 2023 12:02:51 -0700 Subject: [PATCH 06/12] Use actual code context for enumerating (#1721) --- Src/IronPython.Modules/IterTools.cs | 26 ++--- Src/IronPython.Modules/_ssl.cs | 2 +- Src/IronPython.Modules/grp.cs | 2 +- Src/IronPython.Modules/xxsubtype.cs | 4 +- Src/IronPython.SQLite/Cursor.cs | 2 +- Src/IronPython/Modules/Builtin.cs | 4 +- Src/IronPython/Modules/_ast.cs | 4 +- .../Runtime/Binding/ConversionBinder.cs | 7 +- .../Runtime/Binding/MetaPythonFunction.cs | 1 + Src/IronPython/Runtime/Bytes.cs | 2 +- Src/IronPython/Runtime/ClrModule.cs | 8 +- Src/IronPython/Runtime/Enumerate.cs | 70 ++++++------ .../Runtime/Operations/InstanceOps.cs | 2 +- .../Runtime/Operations/PythonOps.cs | 24 ++-- Src/IronPython/Runtime/PythonContext.cs | 4 +- Src/IronPython/Runtime/PythonList.cs | 104 ++++++++++++------ Src/IronPython/Runtime/Set.cs | 2 +- Src/IronPython/Runtime/Types/PythonType.cs | 4 +- 18 files changed, 160 insertions(+), 112 deletions(-) diff --git a/Src/IronPython.Modules/IterTools.cs b/Src/IronPython.Modules/IterTools.cs index f0efd9155..5de12db3d 100644 --- a/Src/IronPython.Modules/IterTools.cs +++ b/Src/IronPython.Modules/IterTools.cs @@ -743,11 +743,11 @@ private static Exception UnexpectedKeywordArgument(IDictionary p [PythonType] public class product : IterBase { - public product(params object[] iterables) { - InnerEnumerator = Yielder(ArrayUtils.ConvertAll(iterables, x => new PythonList(PythonOps.GetEnumerator(x)))); + public product(CodeContext context, params object[] iterables) { + InnerEnumerator = Yielder(ArrayUtils.ConvertAll(iterables, x => new PythonList(context, PythonOps.GetEnumerator(x)))); } - public product([ParamDictionary]IDictionary paramDict, params object[] iterables) { + public product(CodeContext context, [ParamDictionary]IDictionary paramDict, params object[] iterables) { object repeat; int iRepeat = 1; if (paramDict.TryGetValue("repeat", out repeat)) { @@ -768,7 +768,7 @@ public product([ParamDictionary]IDictionary paramDict, params ob PythonList[] finalIterables = new PythonList[iterables.Length * iRepeat]; for (int i = 0; i < iRepeat; i++) { for (int j = 0; j < iterables.Length; j++) { - finalIterables[i * iterables.Length + j] = new PythonList(iterables[j]); + finalIterables[i * iterables.Length + j] = new PythonList(context, iterables[j]); } } InnerEnumerator = Yielder(finalIterables); @@ -823,8 +823,8 @@ private IEnumerator Yielder(PythonList[] iterables) { public class combinations : IterBase { private readonly PythonList _data; - public combinations(object iterable, object r) { - _data = new PythonList(iterable); + public combinations(CodeContext context, object iterable, object r) { + _data = new PythonList(context, iterable); InnerEnumerator = Yielder(GetR(r, _data)); } @@ -893,8 +893,8 @@ private IEnumerator Yielder(int r) { public class combinations_with_replacement : IterBase { private readonly PythonList _data; - public combinations_with_replacement(object iterable, object r) { - _data = new PythonList(iterable); + public combinations_with_replacement(CodeContext context, object iterable, object r) { + _data = new PythonList(context, iterable); InnerEnumerator = Yielder(GetR(r, _data)); } @@ -962,14 +962,14 @@ private IEnumerator Yielder(int r) { public class permutations : IterBase { private readonly PythonList _data; - public permutations(object iterable) { - _data = new PythonList(iterable); + public permutations(CodeContext context, object iterable) { + _data = new PythonList(context, iterable); InnerEnumerator = Yielder(_data.Count); } - public permutations(object iterable, object r) { - _data = new PythonList(iterable); + public permutations(CodeContext context, object iterable, object r) { + _data = new PythonList(context, iterable); InnerEnumerator = Yielder(GetR(r, _data)); } @@ -1160,7 +1160,7 @@ private IEnumerator Yielder(CodeContext context, object function, IEnume objargs[i] = args[i]; } } else { - PythonList argsList = new PythonList(PythonOps.GetEnumerator(iter.Current)); + PythonList argsList = new PythonList(context, PythonOps.GetEnumerator(iter.Current)); objargs = ArrayUtils.ToArray(argsList); } diff --git a/Src/IronPython.Modules/_ssl.cs b/Src/IronPython.Modules/_ssl.cs index 2c202d531..02c871579 100644 --- a/Src/IronPython.Modules/_ssl.cs +++ b/Src/IronPython.Modules/_ssl.cs @@ -208,7 +208,7 @@ public void load_cert_chain(CodeContext context, string certfile, string keyfile public PythonList get_ca_certs(CodeContext context, bool binary_form = false) { if (binary_form) throw new NotImplementedException(nameof(binary_form)); - return new PythonList(_cert_store.Cast().Select(c => CertificateToPython(context, c))); + return PythonList.FromEnumerable(_cert_store.Cast().Select(c => CertificateToPython(context, c))); } public void load_verify_locations(CodeContext context, object cafile = null, string capath = null, object cadata = null) { diff --git a/Src/IronPython.Modules/grp.cs b/Src/IronPython.Modules/grp.cs index 15e233316..80f3a4520 100644 --- a/Src/IronPython.Modules/grp.cs +++ b/Src/IronPython.Modules/grp.cs @@ -76,7 +76,7 @@ internal struct_group(string gr_name, string gr_passwd, int gr_gid, PythonList g private static struct_group Make(IntPtr pwd) { group g = (group)Marshal.PtrToStructure(pwd, typeof(group)); - return new struct_group(g.gr_name, g.gr_passwd, g.gr_gid, new PythonList(MarshalStringArray(g.gr_mem))); + return new struct_group(g.gr_name, g.gr_passwd, g.gr_gid, PythonList.FromEnumerable(MarshalStringArray(g.gr_mem))); } private static IEnumerable MarshalStringArray(IntPtr arrayPtr) diff --git a/Src/IronPython.Modules/xxsubtype.cs b/Src/IronPython.Modules/xxsubtype.cs index 031e092e8..8980fc8c1 100644 --- a/Src/IronPython.Modules/xxsubtype.cs +++ b/Src/IronPython.Modules/xxsubtype.cs @@ -24,8 +24,8 @@ public spamlist() : base() { } - public spamlist(object sequence) - : base(sequence) { + public spamlist(CodeContext context, object sequence) + : base(context, sequence) { } private int _state; diff --git a/Src/IronPython.SQLite/Cursor.cs b/Src/IronPython.SQLite/Cursor.cs index 772429d7c..8570ad7ef 100644 --- a/Src/IronPython.SQLite/Cursor.cs +++ b/Src/IronPython.SQLite/Cursor.cs @@ -121,7 +121,7 @@ private object queryExecute(CodeContext context, bool multiple, object operation if(multiple) { if(args != null) - parameters_iter = PythonOps.CreatePythonEnumerator(args); + parameters_iter = PythonOps.CreatePythonEnumerator(context, args); } else { diff --git a/Src/IronPython/Modules/Builtin.cs b/Src/IronPython/Modules/Builtin.cs index f2895ac8c..3dad01e67 100644 --- a/Src/IronPython/Modules/Builtin.cs +++ b/Src/IronPython/Modules/Builtin.cs @@ -250,7 +250,7 @@ public static void delattr(CodeContext/*!*/ context, object? o, [NotNone] string public static PythonType dict => TypeCache.Dict; public static PythonList dir(CodeContext/*!*/ context) { - PythonList res = new PythonList(context.Dict.Keys); + PythonList res = new PythonList(context, context.Dict.Keys); res.Sort(context); return res; @@ -258,7 +258,7 @@ public static PythonList dir(CodeContext/*!*/ context) { public static PythonList dir(CodeContext/*!*/ context, object? o) { IList ret = PythonOps.GetAttrNames(context, o); - PythonList lret = new PythonList(ret); + PythonList lret = new PythonList(context, ret); lret.Sort(context); return lret; } diff --git a/Src/IronPython/Modules/_ast.cs b/Src/IronPython/Modules/_ast.cs index 679e1e70e..42d5deabc 100755 --- a/Src/IronPython/Modules/_ast.cs +++ b/Src/IronPython/Modules/_ast.cs @@ -1773,7 +1773,7 @@ public Global(PythonList names, [Optional] int? lineno, [Optional] int? col_offs internal Global(GlobalStatement stmt) : this() { - names = new PythonList(stmt.Names); + names = PythonList.FromGenericCollection(stmt.Names); } internal override Statement Revert() { @@ -2345,7 +2345,7 @@ public Nonlocal(PythonList names, [Optional] int? lineno, [Optional] int? col_of internal Nonlocal(NonlocalStatement stmt) : this() { - names = new PythonList(stmt.Names); + names = PythonList.FromGenericCollection(stmt.Names); } internal override Statement Revert() { diff --git a/Src/IronPython/Runtime/Binding/ConversionBinder.cs b/Src/IronPython/Runtime/Binding/ConversionBinder.cs index 507e395a6..61349b7aa 100644 --- a/Src/IronPython/Runtime/Binding/ConversionBinder.cs +++ b/Src/IronPython/Runtime/Binding/ConversionBinder.cs @@ -781,7 +781,7 @@ internal static DynamicMetaObject ConvertToIEnumerable(DynamicMetaObjectBinder/* PythonTypeSlot pts; if (pt.TryResolveSlot(context, "__iter__", out pts)) { - return MakeIterRule(metaUserObject, nameof(PythonOps.CreatePythonEnumerable)); + return MakeIterRule(metaUserObject, pyContext, nameof(PythonOps.CreatePythonEnumerable)); } else if (pt.TryResolveSlot(context, "__getitem__", out pts)) { return MakeGetItemIterable(metaUserObject, pyContext, pts, nameof(PythonOps.CreateItemEnumerable)); } @@ -804,6 +804,7 @@ internal static DynamicMetaObject ConvertToIEnumerator(DynamicMetaObjectBinder/* new[] { tmp }, Expression.Call( typeof(PythonOps).GetMethod(nameof(PythonOps.CreatePythonEnumerator)), + AstUtils.Constant(context), Ast.Block( MetaPythonObject.MakeTryGetTypeMember( state, @@ -839,6 +840,7 @@ private static DynamicMetaObject MakeGetItemIterable(DynamicMetaObject metaUserO new[] { tmp }, Expression.Call( typeof(PythonOps).GetMethod(method), + AstUtils.Constant(state.SharedContext), AstUtils.Convert(metaUserObject.Expression, typeof(object)), Ast.Block( MetaPythonObject.MakeTryGetTypeMember( @@ -867,10 +869,11 @@ private static DynamicMetaObject MakeGetItemIterable(DynamicMetaObject metaUserO ); } - private static DynamicMetaObject/*!*/ MakeIterRule(DynamicMetaObject/*!*/ self, string methodName) { + private static DynamicMetaObject/*!*/ MakeIterRule(DynamicMetaObject/*!*/ self, PythonContext state, string methodName) { return new DynamicMetaObject( Ast.Call( typeof(PythonOps).GetMethod(methodName), + AstUtils.Constant(state.SharedContext), AstUtils.Convert(self.Expression, typeof(object)) ), self.Restrictions diff --git a/Src/IronPython/Runtime/Binding/MetaPythonFunction.cs b/Src/IronPython/Runtime/Binding/MetaPythonFunction.cs index 245afa076..cd492f595 100644 --- a/Src/IronPython/Runtime/Binding/MetaPythonFunction.cs +++ b/Src/IronPython/Runtime/Binding/MetaPythonFunction.cs @@ -955,6 +955,7 @@ private void MakeParamsCopy(Expression/*!*/ userList) { _params, Ast.Call( typeof(PythonOps).GetMethod(nameof(PythonOps.CopyAndVerifyParamsList)), + _codeContext ?? AstUtils.Constant(DefaultContext.Default), AstUtils.Convert(GetFunctionParam(), typeof(PythonFunction)), AstUtils.Convert(userList, typeof(object)) ) diff --git a/Src/IronPython/Runtime/Bytes.cs b/Src/IronPython/Runtime/Bytes.cs index 034290cc3..1bb46f213 100644 --- a/Src/IronPython/Runtime/Bytes.cs +++ b/Src/IronPython/Runtime/Bytes.cs @@ -1165,7 +1165,7 @@ public IEnumerator GetEnumerator() { System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { // workaround for https://github.com/IronLanguages/ironpython3/issues/1519 if (GetType() != typeof(Bytes) && PythonTypeOps.TryInvokeUnaryOperator(DefaultContext.Default, this, "__iter__", out object? iter)) { - return new PythonEnumerator(iter); + return new PythonEnumerator(DefaultContext.Default, iter); } return _bytes.GetEnumerator(); } diff --git a/Src/IronPython/Runtime/ClrModule.cs b/Src/IronPython/Runtime/ClrModule.cs index 0930d6620..7e7b404bc 100644 --- a/Src/IronPython/Runtime/ClrModule.cs +++ b/Src/IronPython/Runtime/ClrModule.cs @@ -837,9 +837,9 @@ private static bool IsInstanceOf(object obj, PythonType pt) { /// /// returns the result of dir(o) as-if "import clr" has not been performed. /// - public static PythonList Dir(object o) { + public static PythonList Dir(CodeContext context, object o) { IList ret = PythonOps.GetAttrNames(DefaultContext.Default, o); - PythonList lret = new PythonList(ret); + PythonList lret = new PythonList(context, ret); lret.Sort(DefaultContext.Default); return lret; } @@ -847,9 +847,9 @@ public static PythonList Dir(object o) { /// /// Returns the result of dir(o) as-if "import clr" has been performed. /// - public static PythonList DirClr(object o) { + public static PythonList DirClr(CodeContext context, object o) { IList ret = PythonOps.GetAttrNames(DefaultContext.DefaultCLS, o); - PythonList lret = new PythonList(ret); + PythonList lret = new PythonList(context, ret); lret.Sort(DefaultContext.DefaultCLS); return lret; } diff --git a/Src/IronPython/Runtime/Enumerate.cs b/Src/IronPython/Runtime/Enumerate.cs index 654402cf9..c8fae3449 100644 --- a/Src/IronPython/Runtime/Enumerate.cs +++ b/Src/IronPython/Runtime/Enumerate.cs @@ -28,14 +28,14 @@ public class Enumerate : IEnumerator, IEnumerator { private readonly IEnumerator _iter; private object _index; - public Enumerate(object iter) { - _iter = PythonOps.GetEnumerator(iter); + public Enumerate(CodeContext context, object iter) { + _iter = PythonOps.GetEnumerator(context, iter); _index = ScriptingRuntimeHelpers.Int32ToObject(-1); } public Enumerate(CodeContext context, object iter, object start) { object index = PythonOps.Index(start); - _iter = PythonOps.GetEnumerator(iter); + _iter = PythonOps.GetEnumerator(context, iter); _index = context.LanguageContext.Operation(Binding.PythonOperationKind.Subtract, index, ScriptingRuntimeHelpers.Int32ToObject(1)); } @@ -176,16 +176,17 @@ void IDisposable.Dispose() { [PythonType("enumerator")] public class PythonEnumerator : IEnumerator { + private readonly CodeContext _context; private readonly object _baseObject; private object _current; public static bool TryCastIEnumer(object baseObject, out IEnumerator enumerator) { - if (baseObject is IEnumerator) { - enumerator = (IEnumerator)baseObject; + if (baseObject is IEnumerator et) { + enumerator = et; return true; } - if (baseObject is IEnumerable) { - enumerator = ((IEnumerable)baseObject).GetEnumerator(); + if (baseObject is IEnumerable en) { + enumerator = en.GetEnumerator(); return true; } @@ -193,18 +194,18 @@ public static bool TryCastIEnumer(object baseObject, out IEnumerator enumerator) return false; } - public static bool TryCreate(object baseObject, out IEnumerator enumerator) { + public static bool TryCreate(CodeContext context, object baseObject, out IEnumerator enumerator) { if (TryCastIEnumer(baseObject, out enumerator)) { return true; } - if (PythonOps.TryGetBoundAttr(baseObject, "__iter__", out object iter)) { + if (PythonOps.TryGetBoundAttr(context, baseObject, "__iter__", out object iter)) { object iterator = PythonCalls.Call(iter); // don't re-wrap if we don't need to (common case is PythonGenerator). if (TryCastIEnumer(iterator, out enumerator)) { return true; } - enumerator = new PythonEnumerator(iterator); + enumerator = new PythonEnumerator(context, iterator); return true; } else { enumerator = null; @@ -212,17 +213,18 @@ public static bool TryCreate(object baseObject, out IEnumerator enumerator) { } } - public static IEnumerator Create(object baseObject) { + public static IEnumerator Create(CodeContext context, object baseObject) { IEnumerator res; - if (!TryCreate(baseObject, out res)) { + if (!TryCreate(context, baseObject, out res)) { throw PythonOps.TypeError("cannot convert {0} to IEnumerator", PythonOps.GetPythonTypeName(baseObject)); } return res; } - internal PythonEnumerator(object iter) { + internal PythonEnumerator(CodeContext context, object iter) { Debug.Assert(!(iter is PythonGenerator)); + _context = context; _baseObject = iter; } @@ -244,14 +246,14 @@ public object Current { /// /// True if moving was successfull public bool MoveNext() { - PythonTypeOps.TryGetOperator(DefaultContext.Default, _baseObject, "__next__", out object nextMethod); + PythonTypeOps.TryGetOperator(_context, _baseObject, "__next__", out object nextMethod); if (nextMethod == null) { throw PythonOps.TypeErrorForNotAnIterator(_baseObject); } try { - _current = DefaultContext.Default.LanguageContext.CallLightEh(DefaultContext.Default, nextMethod); + _current = _context.LanguageContext.CallLightEh(_context, nextMethod); Exception lightEh = LightExceptions.GetLightException(_current); if (lightEh != null) { if (lightEh is StopIterationException) { @@ -275,21 +277,22 @@ public object __iter__() { [PythonType("enumerable")] public class PythonEnumerable : IEnumerable { + private readonly CodeContext _context; private readonly object _iterator; - public static bool TryCreate(object baseEnumerator, out IEnumerable enumerator) { + public static bool TryCreate(CodeContext context, object baseEnumerator, out IEnumerable enumerator) { Debug.Assert(!(baseEnumerator is IEnumerable) || baseEnumerator is IPythonObject); // we shouldn't re-wrap things that don't need it - if (PythonOps.TryGetBoundAttr(baseEnumerator, "__iter__", out object iter)) { - object iterator = PythonCalls.Call(iter); - if (iterator is IEnumerable) { - enumerator = (IEnumerable)iterator; + if (PythonOps.TryGetBoundAttr(context, baseEnumerator, "__iter__", out object iter)) { + object iterator = PythonCalls.Call(context, iter); + if (iterator is IEnumerable en) { + enumerator = en; } else { - if (!PythonOps.TryGetBoundAttr(iterator, "__next__", out _)) { + if (!PythonOps.TryGetBoundAttr(context, iterator, "__next__", out _)) { enumerator = null; return false; } - enumerator = new PythonEnumerable(iterator); + enumerator = new PythonEnumerable(context, iterator); } return true; } else { @@ -298,22 +301,23 @@ public static bool TryCreate(object baseEnumerator, out IEnumerable enumerator) } } - public static IEnumerable Create(object baseObject) { + public static IEnumerable Create(CodeContext context, object baseObject) { IEnumerable res; - if (!TryCreate(baseObject, out res)) { + if (!TryCreate(context, baseObject, out res)) { throw PythonOps.TypeError("cannot convert {0} to IEnumerable", PythonOps.GetPythonTypeName(baseObject)); } return res; } - private PythonEnumerable(object iterator) { - this._iterator = iterator; + private PythonEnumerable(CodeContext context, object iterator) { + _iterator = iterator; + _context = iterator is IEnumerable ? DefaultContext.Default : context; } #region IEnumerable Members IEnumerator IEnumerable.GetEnumerator() { - return _iterator as IEnumerator ?? new PythonEnumerator(_iterator); + return _iterator as IEnumerator ?? new PythonEnumerator(_context, _iterator); } #endregion @@ -321,6 +325,7 @@ IEnumerator IEnumerable.GetEnumerator() { [PythonType("iterator")] public sealed class ItemEnumerator : IEnumerator { + private readonly CodeContext _context; // The actual object on which we are calling __getitem__() private object _source; private object _getItemMethod; @@ -328,7 +333,8 @@ public sealed class ItemEnumerator : IEnumerator { private object _current; private int _index; - internal ItemEnumerator(object source, object getItemMethod, CallSite> site) { + internal ItemEnumerator(CodeContext context, object source, object getItemMethod, CallSite> site) { + _context = context; _source = source; _getItemMethod = getItemMethod; _site = site; @@ -374,7 +380,7 @@ bool IEnumerator.MoveNext() { } try { - _current = _site.Target(_site, DefaultContext.Default, _getItemMethod, _index); + _current = _site.Target(_site, _context, _getItemMethod, _index); _index++; return true; } catch (IndexOutOfRangeException) { @@ -404,11 +410,13 @@ void IEnumerator.Reset() { [PythonType("iterable")] public sealed class ItemEnumerable : IEnumerable { + private readonly CodeContext _context; private readonly object _source; private readonly object _getitem; private readonly CallSite> _site; - internal ItemEnumerable(object source, object getitem, CallSite> site) { + internal ItemEnumerable(CodeContext context, object source, object getitem, CallSite> site) { + _context = context; _source = source; _getitem = getitem; _site = site; @@ -421,7 +429,7 @@ public IEnumerator __iter__() { #region IEnumerable Members IEnumerator IEnumerable.GetEnumerator() { - return new ItemEnumerator(_source, _getitem, _site); + return new ItemEnumerator(_context, _source, _getitem, _site); } #endregion diff --git a/Src/IronPython/Runtime/Operations/InstanceOps.cs b/Src/IronPython/Runtime/Operations/InstanceOps.cs index 52c018892..2b6c6578f 100644 --- a/Src/IronPython/Runtime/Operations/InstanceOps.cs +++ b/Src/IronPython/Runtime/Operations/InstanceOps.cs @@ -212,7 +212,7 @@ public static object NextMethod(object self) { /// __dir__(self) -> Returns the list of members defined on a foreign IDynamicMetaObjectProvider. /// public static PythonList DynamicDir(CodeContext/*!*/ context, IDynamicMetaObjectProvider self) { - PythonList res = new PythonList(self.GetMetaObject(Expression.Parameter(typeof(object))).GetDynamicMemberNames()); + PythonList res = new PythonList(context, self.GetMetaObject(Expression.Parameter(typeof(object))).GetDynamicMemberNames()); // add in the non-dynamic members from the dynamic objects base class. Type t = self.GetType(); diff --git a/Src/IronPython/Runtime/Operations/PythonOps.cs b/Src/IronPython/Runtime/Operations/PythonOps.cs index c8f7e7ef9..cd2d757b2 100644 --- a/Src/IronPython/Runtime/Operations/PythonOps.cs +++ b/Src/IronPython/Runtime/Operations/PythonOps.cs @@ -1063,7 +1063,7 @@ internal static IList GetStringMemberList(IPythonMembersList pyMemList) } if (o is IMembersList memList) { - return new PythonList(memList.GetMemberNames()); + return new PythonList(context, memList.GetMemberNames()); } if (o is IPythonObject po) { @@ -1761,7 +1761,7 @@ public static void DictUpdate(CodeContext context, PythonDictionary dict, object /// LIST_EXTEND /// [EditorBrowsable(EditorBrowsableState.Never)] - public static void ListExtend(PythonList list, object? o) => list.extend(o); + public static void ListExtend(PythonList list, object? o) => list.extend(DefaultContext.Default, o); /// /// LIST_TO_TUPLE @@ -2634,8 +2634,8 @@ public static void VerifyUnduplicatedByName(PythonFunction function, string name } - public static PythonList CopyAndVerifyParamsList(PythonFunction function, object list) { - return new PythonList(list); + public static PythonList CopyAndVerifyParamsList(CodeContext context, PythonFunction function, object list) { + return new PythonList(context, list); } public static PythonTuple UserMappingToPythonTuple(CodeContext/*!*/ context, object list, string funcName) { @@ -3238,24 +3238,24 @@ public static DynamicMetaObjectBinder MakeSimpleCallAction(int count) { return ((PythonGenerator)self).CheckThrowableAndReturnSendValue(); } - public static ItemEnumerable CreateItemEnumerable(object source, object callable, CallSite> site) { - return new ItemEnumerable(source, callable, site); + public static ItemEnumerable CreateItemEnumerable(CodeContext context, object source, object callable, CallSite> site) { + return new ItemEnumerable(context, source, callable, site); } public static DictionaryKeyEnumerator MakeDictionaryKeyEnumerator(PythonDictionary dict) { return new DictionaryKeyEnumerator(dict._storage); } - public static IEnumerable CreatePythonEnumerable(object baseObject) { - return PythonEnumerable.Create(baseObject); + public static IEnumerable CreatePythonEnumerable(CodeContext context, object baseObject) { + return PythonEnumerable.Create(context, baseObject); } - public static IEnumerator CreateItemEnumerator(object source, object callable, CallSite> site) { - return new ItemEnumerator(source, callable, site); + public static IEnumerator CreateItemEnumerator(CodeContext context, object source, object callable, CallSite> site) { + return new ItemEnumerator(context, source, callable, site); } - public static IEnumerator CreatePythonEnumerator(object baseObject) { - return PythonEnumerator.Create(baseObject); + public static IEnumerator CreatePythonEnumerator(CodeContext context, object baseObject) { + return PythonEnumerator.Create(context, baseObject); } public static bool ContainsFromEnumerable(CodeContext/*!*/ context, object enumerable, object value) { diff --git a/Src/IronPython/Runtime/PythonContext.cs b/Src/IronPython/Runtime/PythonContext.cs index cee5a1b27..d13d72a75 100644 --- a/Src/IronPython/Runtime/PythonContext.cs +++ b/Src/IronPython/Runtime/PythonContext.cs @@ -1260,7 +1260,7 @@ private void UnhookAssemblyResolve() { public override ICollection GetSearchPaths() { List result = new List(); if (TryGetSystemPath(out PythonList paths)) { - IEnumerator ie = PythonOps.GetEnumerator(paths); + IEnumerator ie = PythonOps.GetEnumerator(SharedContext, paths); while (ie.MoveNext()) { if (TryConvertToString(ie.Current, out string str)) { result.Add(str); @@ -1271,7 +1271,7 @@ public override ICollection GetSearchPaths() { } public override void SetSearchPaths(ICollection paths) { - SetSystemStateValue("path", new PythonList(paths)); + SetSystemStateValue("path", new PythonList(SharedContext, paths)); } public override void Shutdown() { diff --git a/Src/IronPython/Runtime/PythonList.cs b/Src/IronPython/Runtime/PythonList.cs index 0b1527803..94c525199 100644 --- a/Src/IronPython/Runtime/PythonList.cs +++ b/Src/IronPython/Runtime/PythonList.cs @@ -33,6 +33,8 @@ public class PythonList : IList, ICodeFormattable, IList, IReversible, internal int _size; internal volatile object?[] _data; + #region Python Constructors and Initializers + public void __init__() { _data = new object[8]; _size = 0; @@ -106,7 +108,7 @@ public void __init__(CodeContext context, object? sequence) { _data = new object[len]; _size = 0; - ExtendNoLengthCheck(sequence); + ExtendNoLengthCheck(context, sequence); } public static object __new__(CodeContext/*!*/ context, [NotNone] PythonType cls) { @@ -126,9 +128,30 @@ public static object __new__(CodeContext/*!*/ context, [NotNone] PythonType cls, public static object __new__(CodeContext/*!*/ context, [NotNone] PythonType cls, [ParamDictionary, NotNone] IDictionary kwArgs\u00F8, [NotNone] params object[] args\u00F8) => __new__(context, cls); - private PythonList(IEnumerator e) - : this(10) { - while (e.MoveNext()) AddNoLock(e.Current); + #endregion + + #region C# Constructors and Factories + + public PythonList() + : this(0) { + } + + public PythonList(CodeContext context, [NotNone] object sequence) { + if (sequence is ICollection items) { + _data = new object[items.Count]; + int i = 0; + foreach (object? item in items) { + _data[i++] = item; + } + _size = i; + } else { + if (!PythonOps.TryInvokeLengthHint(context, sequence, out int len)) { + len = INITIAL_SIZE; + } + + _data = new object[len]; + ExtendNoLengthCheck(context, sequence); + } } internal PythonList(int capacity) { @@ -139,13 +162,24 @@ internal PythonList(int capacity) { } } - private PythonList(params object?[] items) { + internal PythonList(ICollection items) + : this(items.Count) { + + int i = 0; + foreach (object? item in items) { + _data[i++] = item; + } + _size = i; + } + + private PythonList(object?[] items) { _data = items; _size = _data.Length; } - public PythonList() - : this(0) { + private PythonList(IEnumerator e) + : this(10) { + while (e.MoveNext()) AddNoLock(e.Current); } #if ALLOC_DEBUG @@ -160,32 +194,24 @@ public PythonList() } #endif - internal PythonList(object sequence) { - if (sequence is ICollection items) { - _data = new object[items.Count]; - int i = 0; - foreach (object? item in items) { - _data[i++] = item; - } - _size = i; - } else { - if (!PythonOps.TryInvokeLengthHint(DefaultContext.Default, sequence, out int len)) { - len = INITIAL_SIZE; - } + internal static PythonList FromGenericCollection(ICollection items) { + var list = new PythonList(items.Count); - _data = new object[len]; - ExtendNoLengthCheck(sequence); + int i = 0; + foreach (object? item in items) { + list._data[i++] = item; } + list._size = i; + return list; } - internal PythonList(ICollection items) - : this(items.Count) { - - int i = 0; - foreach (object? item in items) { - _data[i++] = item; + internal static PythonList FromEnumerable(IEnumerable items) { + var enumerator = items.GetEnumerator(); + try { + return new PythonList(enumerator); + } finally { + (enumerator as IDisposable)?.Dispose(); } - _size = i; } /// @@ -197,6 +223,8 @@ internal PythonList(ICollection items) internal static PythonList FromArrayNoCopy(params object[] data) => new PythonList(data); + #endregion + internal object?[] GetObjectArray() { lock (this) { return ArrayOps.CopyArray(_data, _size); @@ -445,7 +473,7 @@ public virtual object? this[[NotNone] Slice slice] { set { if (slice.step != null && (!(slice.step is int) || !slice.step.Equals(_boxedOne))) { // try to assign back to self: make a copy first - if (this == value) value = new PythonList(value); + if (this == value) value = new PythonList((ICollection)value); if (ValueRequiresNoLocks(value)) { // we don't need to worry about lock ordering of accesses to the @@ -743,16 +771,24 @@ public void extend([NotNone] PythonTuple/*!*/ seq) { } } - public void extend(object? seq) { - if (PythonOps.TryInvokeLengthHint(DefaultContext.Default, seq, out int len)) { + public void extend(CodeContext context, object? seq) { + if (PythonOps.TryInvokeLengthHint(context, seq, out int len)) { EnsureSize(len); } - ExtendNoLengthCheck(seq); + ExtendNoLengthCheck(context, seq); + } + + internal void ExtendNoLock(ICollection seq) { + EnsureSize(Count + seq.Count); + + foreach (var item in seq) { + AddNoLock(item); + } } - private void ExtendNoLengthCheck(object? seq) { - IEnumerator i = PythonOps.GetEnumerator(seq); + private void ExtendNoLengthCheck(CodeContext context, object? seq) { + IEnumerator i = PythonOps.GetEnumerator(context, seq); if (seq == (object)this) { PythonList other = new PythonList(i); i = ((IEnumerable)other).GetEnumerator(); diff --git a/Src/IronPython/Runtime/Set.cs b/Src/IronPython/Runtime/Set.cs index 487c7ec18..4fee48acd 100644 --- a/Src/IronPython/Runtime/Set.cs +++ b/Src/IronPython/Runtime/Set.cs @@ -1477,7 +1477,7 @@ public PythonTuple __reduce__(CodeContext/*!*/ context) { context.TryLookupBuiltin("iter", out iter); if (_cnt < 0) return PythonTuple.MakeTuple(iter, PythonTuple.MakeTuple(new PythonList())); - return PythonTuple.MakeTuple(iter, PythonTuple.MakeTuple(new PythonList(_items)), _cnt); + return PythonTuple.MakeTuple(iter, PythonTuple.MakeTuple(new PythonList(context, _items)), _cnt); } public int __length_hint__() { diff --git a/Src/IronPython/Runtime/Types/PythonType.cs b/Src/IronPython/Runtime/Types/PythonType.cs index 53688c208..6e774c591 100644 --- a/Src/IronPython/Runtime/Types/PythonType.cs +++ b/Src/IronPython/Runtime/Types/PythonType.cs @@ -1747,7 +1747,7 @@ private PythonList TryGetCustomDir(CodeContext context, object self) { ? context.LanguageContext.GetSiteCacheForSystemType(UnderlyingSystemType).GetDirSite(context) : _siteCache.GetDirSite(context); - return new PythonList(dirSite.Target(dirSite, context, dir)); + return new PythonList(context, dirSite.Target(dirSite, context, dir)); } } @@ -1790,7 +1790,7 @@ private static PythonList AddInstanceMembers(object self, Dictionary strKeys = new List(keys.Keys); strKeys.Sort(); - res.extend(strKeys); + res.ExtendNoLock(strKeys); return res; } From 72f4268ae0bb09980a858cfd84b51644b004bf8c Mon Sep 17 00:00:00 2001 From: slozier Date: Fri, 8 Sep 2023 21:17:09 -0400 Subject: [PATCH 07/12] Fix issue with timestamp timezone (#1740) * Fix issue with timestamp timezone * Add test --- Src/IronPython.Modules/_datetime.cs | 2 +- Tests/test_datetime.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Src/IronPython.Modules/_datetime.cs b/Src/IronPython.Modules/_datetime.cs index 2acd15484..b3acc4d42 100644 --- a/Src/IronPython.Modules/_datetime.cs +++ b/Src/IronPython.Modules/_datetime.cs @@ -1036,7 +1036,7 @@ public override string strftime(CodeContext/*!*/ context, [NotNone] string dateF public double timestamp() { if (tzinfo is null) { - return PythonTime.TicksToTimestamp(_dateTime.Ticks); + return PythonTime.TicksToTimestamp(_dateTime.ToUniversalTime().Ticks); } else { return (this - new datetime(new DateTime(1970, 1, 1), timezone.utc)).total_seconds(); diff --git a/Tests/test_datetime.py b/Tests/test_datetime.py index 9faca7fe6..c23225566 100644 --- a/Tests/test_datetime.py +++ b/Tests/test_datetime.py @@ -126,6 +126,11 @@ def test_fromtimestamp(self): ts = 5399410716.777882 self.assertEqual(datetime.datetime.fromtimestamp(ts).microsecond, 777882) + def test_timestamp(self): + # https://github.com/IronLanguages/ironpython3/pull/1740 + dt = datetime.datetime(2000, 1, 1) + self.assertEqual(datetime.datetime.fromtimestamp(dt.timestamp()), dt) + @skipUnlessIronPython() def test_System_DateTime_conversion(self): import clr From f7298e56a80233b9df7b4bf3c546eb21b50d488c Mon Sep 17 00:00:00 2001 From: slozier Date: Thu, 14 Sep 2023 20:00:59 -0400 Subject: [PATCH 08/12] Drop .NET Core 3.1 (#1741) * Drop .NET Core 3.1 * Fix interop tests * Fix sqlite3 test * Try ubuntu-latest * Bump to ubuntu-latest * Update readme --- .editorconfig | 1 + .github/workflows/main.yml | 11 ++--------- .vsts-ci.yml | 2 +- Build/After.targets | 9 ++------- Build/steps.yml | 6 ------ IronPython.sln | 2 -- Package/nuget/IronPython.nuspec | 13 ++++--------- Package/zip/Zip.Packaging.targets | 2 +- README.md | 2 +- Src/DLR | 2 +- .../IronPython.Modules.csproj | 6 +----- Src/IronPython.Modules/signal.cs | 8 ++++---- Src/IronPython.SQLite/IronPython.SQLite.csproj | 12 ++++++++++-- Src/IronPython.Wpf/IronPython.Wpf.csproj | 11 +++++++++-- Src/IronPython/Hosting/PythonCommandLine.cs | 15 ++++++++------- Src/IronPython/IronPython.csproj | 4 ++-- Src/IronPython/Runtime/ConversionWrappers.cs | 10 +--------- Src/IronPythonConsole/IronPythonConsole.csproj | 18 +++++++----------- Src/IronPythonConsole/ipy.bat | 2 -- Src/IronPythonTest/Cases/CaseExecuter.cs | 13 ++++++++++--- Src/IronPythonTest/IronPythonTest.csproj | 10 +++++----- Src/Scripts/Install-IronPython.ps1 | 1 - Tests/test_sqlite3_stdlib.py | 8 +++----- make.ps1 | 2 +- 24 files changed, 74 insertions(+), 96 deletions(-) delete mode 100644 Src/IronPythonConsole/ipy.bat diff --git a/.editorconfig b/.editorconfig index 97bf7d90e..d9c7ff3d7 100644 --- a/.editorconfig +++ b/.editorconfig @@ -82,6 +82,7 @@ dotnet_diagnostic.CA1846.severity = none # CA1846: Prefer 'AsSpan' over ' dotnet_diagnostic.CA1847.severity = none # CA1847: Use char literal for a single character lookup dotnet_diagnostic.CA1852.severity = suggestion # CA1852: Seal internal types dotnet_diagnostic.CA1859.severity = suggestion # CA1859: Use concrete types when possible for improved performance +dotnet_diagnostic.CA1861.severity = suggestion # CA1861: Avoid constant arrays as arguments dotnet_diagnostic.CA2101.severity = suggestion # CA2101: Specify marshaling for P/Invoke string arguments dotnet_diagnostic.CA2201.severity = none # CA2201: Do not raise reserved exception types dotnet_diagnostic.CA2208.severity = suggestion # CA2208: Instantiate argument exceptions correctly diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d1d39654e..0ddab2b28 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,19 +10,15 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-20.04, macos-latest] + os: [windows-latest, ubuntu-latest, macos-latest] steps: - name: Install tools - if: matrix.os == 'ubuntu-20.04' + if: matrix.os == 'ubuntu-latest' run: sudo apt-get -yq install mono-vbnc dos2unix - uses: actions/checkout@v2 with: submodules: true - - name: Setup .NET Core 2.1 - uses: actions/setup-dotnet@v1 - with: - dotnet-version: '2.1.x' - name: Setup .NET Core 3.1 uses: actions/setup-dotnet@v1 with: @@ -48,9 +44,6 @@ jobs: - name: Test (net462) run: ./make.ps1 -frameworks net462 test-all shell: pwsh - - name: Test (netcoreapp2.1) - run: ./make.ps1 -frameworks netcoreapp2.1 test-all - shell: pwsh - name: Test (netcoreapp3.1) run: ./make.ps1 -frameworks netcoreapp3.1 test-all shell: pwsh diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 49804c806..6e8567eb4 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -18,7 +18,7 @@ jobs: displayName: Linux (Ubuntu) timeoutInMinutes: 180 pool: - vmImage: ubuntu-20.04 + vmImage: ubuntu-latest steps: - template: Build/steps.yml parameters: diff --git a/Build/After.targets b/Build/After.targets index 10b44848a..6da59981d 100644 --- a/Build/After.targets +++ b/Build/After.targets @@ -19,8 +19,7 @@ - <_TargetFramework>$(TargetFramework) - <_TargetFramework Condition=" $(TargetFramework.EndsWith('-windows')) ">$(TargetFramework.Substring(0, $(TargetFramework.IndexOf('-windows')))) + <_TargetFramework>$(TargetFramework.Replace('-windows', '')) $(StageDir)\$(_TargetFramework) $(StageDir)\$(_TargetFramework)\DLLs @(StageItem) @@ -44,9 +43,5 @@ - - - - - + diff --git a/Build/steps.yml b/Build/steps.yml index 97d269b15..ab6945c4e 100644 --- a/Build/steps.yml +++ b/Build/steps.yml @@ -24,12 +24,6 @@ steps: Write-Host ("##vso[task.setvariable variable=PackageVersion;isSecret=false;isOutput=true;]$PackageVersion") displayName: Grab Package Version - - task: UseDotNet@2 - displayName: Install .NET Core 2.1 runtime for testing - inputs: - packageType: 'runtime' - version: '2.1.x' - - task: UseDotNet@2 displayName: Install .NET Core 3.1 runtime for testing inputs: diff --git a/IronPython.sln b/IronPython.sln index 762507dfd..d02363401 100644 --- a/IronPython.sln +++ b/IronPython.sln @@ -37,8 +37,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{17737ACB Build\net462.props = Build\net462.props Build\net6.0-windows.props = Build\net6.0-windows.props Build\net6.0.props = Build\net6.0.props - Build\netcoreapp2.1.props = Build\netcoreapp2.1.props - Build\netcoreapp3.1.props = Build\netcoreapp3.1.props Build\netstandard2.0.props = Build\netstandard2.0.props Build\steps.yml = Build\steps.yml Build\Tasks.Targets = Build\Tasks.Targets diff --git a/Package/nuget/IronPython.nuspec b/Package/nuget/IronPython.nuspec index 4708b4cb2..3cc2a5d76 100644 --- a/Package/nuget/IronPython.nuspec +++ b/Package/nuget/IronPython.nuspec @@ -25,12 +25,7 @@ This package contains the IronPython interpreter engine. - - - - - - + @@ -41,9 +36,9 @@ This package contains the IronPython interpreter engine. - - - + + + diff --git a/Package/zip/Zip.Packaging.targets b/Package/zip/Zip.Packaging.targets index b9fbbf0de..cc1f64bb8 100644 --- a/Package/zip/Zip.Packaging.targets +++ b/Package/zip/Zip.Packaging.targets @@ -4,7 +4,7 @@ - + diff --git a/README.md b/README.md index 4b35f2bf1..6d53eb37c 100644 --- a/README.md +++ b/README.md @@ -86,4 +86,4 @@ Binaries of IronPython 3 can be downloaded from the [release page](https://githu See the [building document](Documentation/building.md). Since the main development is on Windows, bugs on other platforms may inadvertently be introduced - please report them! ## Supported Platforms -IronPython 3 targets .NET Framework 4.6.2, .NET Standard 2.0, .NET Core 3.1 and .NET 6.0. The support for .NET and .NET Core follow the lifecycle defined on [.NET and .NET Core Support Policy](https://dotnet.microsoft.com/platform/support/policy/dotnet-core). +IronPython 3 targets .NET Framework 4.6.2, .NET Standard 2.0 and .NET 6.0. The support for .NET and .NET Core follow the lifecycle defined on [.NET and .NET Core Support Policy](https://dotnet.microsoft.com/platform/support/policy/dotnet-core). diff --git a/Src/DLR b/Src/DLR index e2a679d68..7b478c32c 160000 --- a/Src/DLR +++ b/Src/DLR @@ -1 +1 @@ -Subproject commit e2a679d68ab5150e97e2debcd4f35bceb0fca584 +Subproject commit 7b478c32cc228c5226920965c6d1ecfaa0cfd1e4 diff --git a/Src/IronPython.Modules/IronPython.Modules.csproj b/Src/IronPython.Modules/IronPython.Modules.csproj index 17538aaea..5a7b3cba0 100644 --- a/Src/IronPython.Modules/IronPython.Modules.csproj +++ b/Src/IronPython.Modules/IronPython.Modules.csproj @@ -1,7 +1,7 @@  - net462;netcoreapp3.1;netstandard2.0;net6.0 + net462;netstandard2.0;net6.0 885063680 true true @@ -33,10 +33,6 @@ - - - - diff --git a/Src/IronPython.Modules/signal.cs b/Src/IronPython.Modules/signal.cs index 0798afa6d..1ae2de2f8 100644 --- a/Src/IronPython.Modules/signal.cs +++ b/Src/IronPython.Modules/signal.cs @@ -118,13 +118,13 @@ public static object getsignal(CodeContext/*!*/ context, int signalnum) { //Negative Scenarios if (signalnum < 1 || signalnum > 22) { throw PythonOps.ValueError("signal number out of range"); - } else if (!GetPythonSignalState(context).PySignalToPyHandler.ContainsKey(signalnum)) { + } else if (GetPythonSignalState(context).PySignalToPyHandler.TryGetValue(signalnum, out object value)) { + //Default + return value; + } else { //Handles the special case of SIG_IGN. This is not really a signal, //but CPython returns null for it any ways return null; - } else { - //Default - return GetPythonSignalState(context).PySignalToPyHandler[signalnum]; } } } diff --git a/Src/IronPython.SQLite/IronPython.SQLite.csproj b/Src/IronPython.SQLite/IronPython.SQLite.csproj index 95e250e07..f00505312 100644 --- a/Src/IronPython.SQLite/IronPython.SQLite.csproj +++ b/Src/IronPython.SQLite/IronPython.SQLite.csproj @@ -1,12 +1,18 @@  - net462;netcoreapp3.1;netstandard2.0;net6.0 + net462;netstandard2.0;net6.0 true SQLITE_DEBUG;TRUE;WIN32;_MSC_VER;SQLITE_ASCII;SQLITE_MEM_POOL;SQLITE_ENABLE_COLUMN_METADATA;SQLITE_OS_WIN;SQLITE_SYSTEM_MALLOC;VDBE_PROFILE_OFF SQLITE_OMIT_AUTHORIZATION;SQLITE_OMIT_DEPRECATED;SQLITE_OMIT_GET_TABLE;SQLITE_OMIT_INCRBLOB;SQLITE_OMIT_LOOKASIDE;SQLITE_OMIT_SHARED_CACHE;SQLITE_OMIT_UTF16;SQLITE_OMIT_WAL $(NoWarn);0168;0169;0414;0618;0649;1587;219;1570 true + + + false + $(BaseIntermediateOutputPath)$(Configuration)\$(TargetFramework)\DLLs + $(BaseOutputPath)\$(TargetFramework)\DLLs + true @@ -18,7 +24,9 @@ - + + + diff --git a/Src/IronPython.Wpf/IronPython.Wpf.csproj b/Src/IronPython.Wpf/IronPython.Wpf.csproj index 23b38c9c6..9d0ff5997 100644 --- a/Src/IronPython.Wpf/IronPython.Wpf.csproj +++ b/Src/IronPython.Wpf/IronPython.Wpf.csproj @@ -1,14 +1,21 @@  - net462;netcoreapp3.1;net6.0-windows + net462;net6.0-windows true true true + + + false + $(BaseOutputPath)\$(TargetFramework.Replace('-windows', ''))\DLLs + true - + + + diff --git a/Src/IronPython/Hosting/PythonCommandLine.cs b/Src/IronPython/Hosting/PythonCommandLine.cs index 2df72148a..3d1a9fee6 100644 --- a/Src/IronPython/Hosting/PythonCommandLine.cs +++ b/Src/IronPython/Hosting/PythonCommandLine.cs @@ -36,9 +36,9 @@ public PythonCommandLine() { protected override string Logo => PythonContext.PythonOptions.Quiet ? null : GetLogoDisplay(); /// - /// Returns the display look for IronPython. - /// - /// The returned string uses This \n instead of Environment.NewLine for it's line seperator + /// Returns the display look for IronPython. + /// + /// The returned string uses This \n instead of Environment.NewLine for it's line seperator /// because it is intended to be outputted through the Python I/O system. /// public static string GetLogoDisplay() { @@ -265,6 +265,7 @@ private void InitializeModules() { if (File.Exists(runner)) { executable = runner; } else { + // TODO: was for .NET Core 2.1, can we drop this? runner = Path.Combine(prefix, name + ".bat"); if (File.Exists(runner)) executable = runner; } @@ -312,7 +313,7 @@ private void InitializeModules() { /// /// Loads any extension DLLs present in sys.prefix\DLLs directory and adds references to them. - /// + /// /// This provides an easy drop-in location for .NET assemblies which should be automatically referenced /// (exposed via import), COM libraries, and pre-compiled Python code. /// @@ -433,7 +434,7 @@ private void RunStartup() { /// Attempts to run a single interaction and handle any language-specific /// exceptions. Base classes can override this and call the base implementation /// surrounded with their own exception handling. - /// + /// /// Returns null if successful and execution should continue, or an exit code. /// private int? TryInteractiveActionWorker() { @@ -457,8 +458,8 @@ private void RunStartup() { } /// - /// Parses a single interactive command and executes it. - /// + /// Parses a single interactive command and executes it. + /// /// Returns null if successful and execution should continue, or the appropiate exit code. /// private int? RunOneInteraction() { diff --git a/Src/IronPython/IronPython.csproj b/Src/IronPython/IronPython.csproj index c91fc81df..65dcb580f 100644 --- a/Src/IronPython/IronPython.csproj +++ b/Src/IronPython/IronPython.csproj @@ -1,7 +1,7 @@  - net462;netcoreapp3.1;netstandard2.0;net6.0 + net462;netstandard2.0;net6.0 879755264 true true @@ -65,7 +65,7 @@ - + diff --git a/Src/IronPython/Runtime/ConversionWrappers.cs b/Src/IronPython/Runtime/ConversionWrappers.cs index ef36804c7..2a61dfd85 100644 --- a/Src/IronPython/Runtime/ConversionWrappers.cs +++ b/Src/IronPython/Runtime/ConversionWrappers.cs @@ -123,10 +123,6 @@ IEnumerator IEnumerable.GetEnumerator() { #endregion } -#if NETCOREAPP3_1 // In netcoreapp3.1, IDictionary has constraint where K : notnull -#nullable disable warnings -#endif - public class DictionaryGenericWrapper : IDictionary { private readonly IDictionary self; // PEP 237: int/long unification (GH #52) @@ -303,10 +299,6 @@ private static bool IsValidKey32(K key, [NotNullWhen(true)] out object? key32) { } } -#if NETCOREAPP3_1 -#nullable enable -#endif - public class IEnumeratorOfTWrapper : IEnumerator { private readonly IEnumerator enumerable; // PEP 237: int/long unification (GH #52) @@ -327,7 +319,7 @@ public T Current { try { return (T)current!; } catch (NullReferenceException nex) { - throw new InvalidCastException(string.Format("Error in IEnumeratorOfTWrapper.Current. Could not cast: from null to {0}", typeof(T).ToString()), nex); + throw new InvalidCastException(string.Format("Error in IEnumeratorOfTWrapper.Current. Could not cast: from null to {0}", typeof(T).ToString()), nex); } catch (InvalidCastException iex) { throw new InvalidCastException(string.Format("Error in IEnumeratorOfTWrapper.Current. Could not cast: from {1} to {0}", typeof(T).ToString(), current!.GetType().ToString()), iex); } diff --git a/Src/IronPythonConsole/IronPythonConsole.csproj b/Src/IronPythonConsole/IronPythonConsole.csproj index 8bbde8148..73024a0c8 100644 --- a/Src/IronPythonConsole/IronPythonConsole.csproj +++ b/Src/IronPythonConsole/IronPythonConsole.csproj @@ -1,8 +1,8 @@  - net462;netcoreapp2.1;netcoreapp3.1;net6.0 - + net462;netcoreapp3.1;net6.0 + false Exe IronPythonConsole @@ -20,15 +20,12 @@ Content PreserveNewest - - - - - PreserveNewest - - + + + false + Content PreserveNewest - + @@ -38,7 +35,6 @@ - diff --git a/Src/IronPythonConsole/ipy.bat b/Src/IronPythonConsole/ipy.bat deleted file mode 100644 index 098af35bd..000000000 --- a/Src/IronPythonConsole/ipy.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -dotnet "%~dp0ipy.dll" %* diff --git a/Src/IronPythonTest/Cases/CaseExecuter.cs b/Src/IronPythonTest/Cases/CaseExecuter.cs index 3f5194efc..d75cbc047 100644 --- a/Src/IronPythonTest/Cases/CaseExecuter.cs +++ b/Src/IronPythonTest/Cases/CaseExecuter.cs @@ -8,6 +8,7 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -31,8 +32,6 @@ private static string Executable { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { runner = Path.Combine(folder, "ipy.exe"); if (File.Exists(runner)) return runner; - runner = Path.Combine(folder, "ipy.bat"); - if (File.Exists(runner)) return runner; } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { runner = Path.Combine(folder, "ipy"); if (File.Exists(runner)) return runner; @@ -82,7 +81,7 @@ internal static string FindRoot() { private static void AddSearchPaths(ScriptEngine engine) { var paths = new List(engine.GetSearchPaths()); - if (!paths.Any(x => x.ToLowerInvariant().Contains("stdlib"))) { + if (!paths.Any(x => x.Contains("stdlib", StringComparison.OrdinalIgnoreCase))) { var root = FindRoot(); if (!string.IsNullOrEmpty(root)) { paths.Insert(0, Path.Combine(root, "Src", "StdLib", "Lib")); @@ -307,4 +306,12 @@ private int GetResult(TestInfo testcase, ScriptEngine engine, ScriptSource sourc } } } + +#if NETFRAMEWORK + internal static class StringExtensions { + public static bool Contains(this string s, string value, StringComparison comparisonType) { + return s.IndexOf(value, comparisonType) >= 0; + } + } +#endif } diff --git a/Src/IronPythonTest/IronPythonTest.csproj b/Src/IronPythonTest/IronPythonTest.csproj index 3907d1cca..118756389 100644 --- a/Src/IronPythonTest/IronPythonTest.csproj +++ b/Src/IronPythonTest/IronPythonTest.csproj @@ -1,8 +1,8 @@  - net462;netcoreapp2.1;netcoreapp3.1;net6.0 - + net462;netcoreapp3.1;net6.0 + false true @@ -18,8 +18,8 @@ - - + + @@ -35,7 +35,7 @@ - + diff --git a/Src/Scripts/Install-IronPython.ps1 b/Src/Scripts/Install-IronPython.ps1 index 016596c42..5692a12b4 100755 --- a/Src/Scripts/Install-IronPython.ps1 +++ b/Src/Scripts/Install-IronPython.ps1 @@ -112,7 +112,6 @@ dotnet (Join-Path $PSScriptRoot ipy.dll) @args chmod +x $ipyPath chmod +x (Join-Path $Path "ipy.sh") Move-Item -Path (Join-Path $Path "ipy.sh") -Destination (Join-Path $Path "ipy") - Remove-Item -Path (Join-Path $Path "ipy.bat") } } elseif ($IsMacOS -or $IsLinux) { # Mono $ipyPath = Join-Path $Path "ipy" diff --git a/Tests/test_sqlite3_stdlib.py b/Tests/test_sqlite3_stdlib.py index d88e0963c..cfff554ab 100644 --- a/Tests/test_sqlite3_stdlib.py +++ b/Tests/test_sqlite3_stdlib.py @@ -6,11 +6,7 @@ ## Run selected tests from sqlite3.test from StdLib ## -from iptest import is_ironpython, generate_suite, run_test, is_linux, is_netcoreapp21 - -if is_netcoreapp21: raise SystemExit # no IronPython.SQLite.dll with .NET Core 2.1 - -import test.test_sqlite +from iptest import is_ironpython, generate_suite, run_test, is_linux import sqlite3.test.dbapi import sqlite3.test.dump @@ -21,6 +17,8 @@ import sqlite3.test.types import sqlite3.test.userfunctions +import test.test_sqlite + def load_tests(loader, standard_tests, pattern): tests = loader.loadTestsFromModule(test.test_sqlite) diff --git a/make.ps1 b/make.ps1 index 2f9867145..f14921289 100755 --- a/make.ps1 +++ b/make.ps1 @@ -4,7 +4,7 @@ Param( [Parameter(Position=1)] [String] $target = "build", [String] $configuration = "Release", - [String[]] $frameworks=@('net462','netcoreapp2.1','netcoreapp3.1','net6.0'), + [String[]] $frameworks=@('net462','netcoreapp3.1','net6.0'), [String] $platform = "x64", [switch] $runIgnored, [int] $jobs = [System.Environment]::ProcessorCount From d547e116fbc2aafbc83457299ec8f8ee113b426c Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Tue, 17 Oct 2023 13:21:00 -0700 Subject: [PATCH 09/12] Clarify version reporting requirements for new issues --- .github/ISSUE_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 018a2c1ed..c235ec2f6 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -29,11 +29,11 @@ https://ironpython.groups.io/g/users. ### Version Information -If you are using the `ipy` console program, provide output of executing `ipy -VV`. If it is a local build, provide also the git hash of the commit used to build IronPython. Additionally, provide the type of the operating system used. +If you are using the `ipy` console program, provide output of executing `ipy -VV`. If it is a local build, provide also the git hash of the commit used to build IronPython. In either case, provide the type of the operating system used. If you are using the IronPython engine embedded in a .NET application, provide the version number of the NuGet package used (or if it is a local build, the git hash of the commit used to build IronPython), and the following info: -* .NET platform used (choice from: .NET, .NET Core, .NET Framework, Mono, Unity), +* .NET platform used (choice from: .NET, .NET Core, .NET Framework, Mono, Unity, Xamarin), * Version of the .NET platform used, * Operating system used, * Value of `sys.version`, from imported module `sys`. From d670a31608ec5a1403f0cd105a20d6e47ee9423b Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Tue, 14 Nov 2023 16:12:13 -0800 Subject: [PATCH 10/12] Fix install script to run with powershell 5.1 --- Src/Scripts/Install-IronPython.ps1 | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Src/Scripts/Install-IronPython.ps1 b/Src/Scripts/Install-IronPython.ps1 index 5692a12b4..8dacf6709 100755 --- a/Src/Scripts/Install-IronPython.ps1 +++ b/Src/Scripts/Install-IronPython.ps1 @@ -98,7 +98,7 @@ if (-not $unzipDir) { } # Copy files into place -Copy-Item -Path (Join-Path $unzipDir $Framework "*") -Destination $Path -Recurse +Copy-Item -Path (Join-Path $unzipDir (Join-Path $Framework "*")) -Destination $Path -Recurse Copy-Item -Path (Join-Path $unzipDir "lib") -Destination $Path -Recurse # Prepare startup scripts @@ -108,6 +108,12 @@ if ($Framework -notlike "net4*") { #!/usr/bin/env pwsh dotnet (Join-Path $PSScriptRoot ipy.dll) @args '@ + if ($PSVersionTable.PSEdition -eq "Desktop" -or $IsWindows) { + $ipyPath = Join-Path $Path "ipy.bat" + Set-Content -Path $ipyPath -Value @' +@dotnet "%~dp0ipy.dll" %* +'@ + } if ($IsMacOS -or $IsLinux) { chmod +x $ipyPath chmod +x (Join-Path $Path "ipy.sh") From 9aa48ffb56e804a13fd42486368ec684aba122d9 Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Tue, 21 Nov 2023 13:31:47 -0800 Subject: [PATCH 11/12] Fix error handling in Install-IronPython.ps1 --- Src/Scripts/Install-IronPython.ps1 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Src/Scripts/Install-IronPython.ps1 b/Src/Scripts/Install-IronPython.ps1 index 8dacf6709..030d93f3f 100755 --- a/Src/Scripts/Install-IronPython.ps1 +++ b/Src/Scripts/Install-IronPython.ps1 @@ -64,12 +64,13 @@ if (-not $ZipFile) { # Script run from within a checked out code base # Locate the zip archive in the standard location of the package target $projectRoot = $PSScriptRoot | Split-Path | Split-Path - $ZipFile = @(Resolve-Path (Join-Path $projectRoot "Package/Release/Packages/IronPython-*/IronPython.3.*.zip")) - if ($ZipFile.Count -gt 1) { - Write-Error "Ambiguous implicit project zip file: $ZipFile" - } elseif ($ZipFile.Count -lt 1) { + $zipFiles = @(Resolve-Path (Join-Path $projectRoot "Package/Release/Packages/IronPython-*/IronPython.3.*.zip")) + if ($zipFiles.Count -gt 1) { + Write-Error (@("Ambiguous implicit project zip files:") + $zipFiles -join "`n") + } elseif ($zipFiles.Count -lt 1) { Write-Error "Missing zip file. Have you run './make package'?" } + $ZipFile = $zipFiles } else { Write-Error "Cannot locate implicit zip file. Provide path to the zip file using '-ZipFile '." } From 0be3c1cf335e1c44a9ee1a9dadbc7af7aa0f6c6b Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Mon, 11 Dec 2023 11:40:23 -0800 Subject: [PATCH 12/12] Resolve some warnings about string comparisons (#1757) --- Src/DLR | 2 +- Src/IronPython/Runtime/Operations/StringOps.cs | 2 +- Src/IronPythonTest/Util/IniParser.cs | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Src/DLR b/Src/DLR index 7b478c32c..872730ab3 160000 --- a/Src/DLR +++ b/Src/DLR @@ -1 +1 @@ -Subproject commit 7b478c32cc228c5226920965c6d1ecfaa0cfd1e4 +Subproject commit 872730ab305d37f67179db8ca778d1cd22984e67 diff --git a/Src/IronPython/Runtime/Operations/StringOps.cs b/Src/IronPython/Runtime/Operations/StringOps.cs index 929faa75e..ca1cf3896 100644 --- a/Src/IronPython/Runtime/Operations/StringOps.cs +++ b/Src/IronPython/Runtime/Operations/StringOps.cs @@ -2079,7 +2079,7 @@ static CodecsInfo() { #if DEBUG foreach (KeyValuePair> kvp in d) { // all codecs should be stored in lowercase because we only look up from lowercase strings - Debug.Assert(kvp.Key.ToLower(CultureInfo.InvariantCulture) == kvp.Key); + Debug.Assert(kvp.Key.Equals(kvp.Key, StringComparison.OrdinalIgnoreCase)); // all codec names should use underscores instead of dashes to match lookup values Debug.Assert(kvp.Key.IndexOf('-') < 0); } diff --git a/Src/IronPythonTest/Util/IniParser.cs b/Src/IronPythonTest/Util/IniParser.cs index a392dd2b2..a5101dfc1 100644 --- a/Src/IronPythonTest/Util/IniParser.cs +++ b/Src/IronPythonTest/Util/IniParser.cs @@ -72,7 +72,8 @@ private static OptionStore Parse(IEnumerable lines) { if (string.IsNullOrEmpty(line)) continue; - if (line.StartsWith("[", StringComparison.Ordinal) && line.EndsWith("]", StringComparison.Ordinal)) { + //if (line.StartsWith('[', StringComparison.Ordinal) && line.EndsWith(']', StringComparison.Ordinal)) { + if (line.Length >= 2 && line[0] == '[' && line[line.Length - 1] == ']') { var sectionName = line.Substring(1, line.Length - 2); if (!options.TryGetValue(sectionName, out currentSection)) { currentSection = new Section();