Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use PosixFileStream for files on POSIX #1855

Merged
merged 9 commits into from
Jan 9, 2025
Merged

Use PosixFileStream for files on POSIX #1855

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
86adef5
Use PosixFileStream for files on POSIX
BCSharp Dec 30, 2024
64d0d4a
Improve non-buffering strategy for FileStream
BCSharp Dec 31, 2024
1cf9ac6
Ignore OSException along with IOException
BCSharp Jan 1, 2025
847db52
Merge branch 'upstream/main' into 'open_file_posix'
BCSharp Jan 6, 2025
215237c
Fix file size checks in constructor
BCSharp Jan 6, 2025
f4cce8a
Implement proper lifetime management of handles
BCSharp Jan 7, 2025
a858fcd
Describe Rules of Engagement
BCSharp Jan 7, 2025
3bce846
Implement mmap.resize on POSIX
BCSharp Jan 7, 2025
91fc9f4
Update after review
BCSharp Jan 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Src/IronPython.Modules/_warnings.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ internal static void showwarning(CodeContext context, object message, PythonType
((TextWriter)file).Write(text);
} // unrecognized file type - warning is lost
}
} catch (IOException) {
} catch (Exception ex) when (ex is IOException or OSException) {
// invalid file - warning is lost
}
}
Expand Down
130 changes: 101 additions & 29 deletions Src/IronPython.Modules/mmap.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

System.Runtime.Serialization is VS trying to be helpful?

using System.Runtime.Versioning;
using System.Text;
using System.Threading;

Expand All @@ -24,6 +26,7 @@
using IronPython.Runtime.Types;

using Microsoft.Scripting.Utils;
using Microsoft.Win32.SafeHandles;

[assembly: PythonModule("mmap", typeof(IronPython.Modules.MmapModule))]
namespace IronPython.Modules {
Expand Down Expand Up @@ -92,6 +95,7 @@ public class MmapDefault : IWeakReferenceable {
private readonly long _offset;
private readonly string _mapName;
private readonly MemoryMappedFileAccess _fileAccess;
private readonly SafeFileHandle _handle; // only used on some POSIX platforms, null otherwise

private volatile bool _isClosed;
private int _refCount = 1;
Expand Down Expand Up @@ -148,46 +152,65 @@ public MmapDefault(CodeContext/*!*/ context, int fileno, long length, string tag

PythonContext pContext = context.LanguageContext;
if (pContext.FileManager.TryGetStreams(fileno, out StreamBox streams)) {
if ((_sourceStream = streams.ReadStream as FileStream) == null) {
throw WindowsError(PythonExceptions._OSError.ERROR_INVALID_HANDLE);
Stream stream = streams.ReadStream;
if (stream is FileStream fs) {
_sourceStream = fs;
} else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) {
// use file descriptor
#if NET8_0_OR_GREATER
CheckFileAccess(_fileAccess, stream);
_handle = new SafeFileHandle((IntPtr)fileno, ownsHandle: false);
_file = MemoryMappedFile.CreateFromFile(_handle, _mapName, length, _fileAccess, HandleInheritability.None, leaveOpen: true);
#else
_handle = new SafeFileHandle((IntPtr)fileno, ownsHandle: false);
FileAccess fileAccess = stream.CanWrite ? stream.CanRead ? FileAccess.ReadWrite : FileAccess.Write : FileAccess.Read;
// This may or may not work on Mono, but on Mono streams.ReadStream is FileStream (unless dupped in some cases)
_sourceStream = new FileStream(_handle, fileAccess);
#endif
}
// otherwise leaves _file as null and _sourceStream as null
} else {
throw PythonOps.OSError(PythonExceptions._OSError.ERROR_INVALID_BLOCK, "Bad file descriptor");
}

if (_fileAccess == MemoryMappedFileAccess.ReadWrite && !_sourceStream.CanWrite) {
throw WindowsError(PythonExceptions._OSError.ERROR_ACCESS_DENIED);
}
if (_file is null) {
// create _file form _sourceStream
if (_sourceStream is null) {
throw WindowsError(PythonExceptions._OSError.ERROR_INVALID_HANDLE);
}

CheckFileAccess(_fileAccess, _sourceStream);

if (length == 0) {
length = _sourceStream.Length;
if (length == 0) {
throw PythonOps.ValueError("cannot mmap an empty file");
}
if (_offset >= length) {
throw PythonOps.ValueError("mmap offset is greater than file size");
length = _sourceStream.Length;
if (length == 0) {
throw PythonOps.ValueError("cannot mmap an empty file");
}
if (_offset >= length) {
throw PythonOps.ValueError("mmap offset is greater than file size");
}
length -= _offset;
}
length -= _offset;
}

long capacity = checked(_offset + length);
long capacity = checked(_offset + length);

// Enlarge the file as needed.
if (capacity > _sourceStream.Length) {
if (_sourceStream.CanWrite) {
_sourceStream.SetLength(capacity);
} else {
throw WindowsError(PythonExceptions._OSError.ERROR_NOT_ENOUGH_MEMORY);
// Enlarge the file as needed.
if (capacity > _sourceStream.Length) {
if (_sourceStream.CanWrite) {
_sourceStream.SetLength(capacity);
} else {
throw WindowsError(PythonExceptions._OSError.ERROR_NOT_ENOUGH_MEMORY);
}
}
}

_file = CreateFromFile(
_sourceStream,
_mapName,
_sourceStream.Length,
_fileAccess,
HandleInheritability.None,
true);
_file = CreateFromFile(
_sourceStream,
_mapName,
_sourceStream.Length,
_fileAccess,
HandleInheritability.None,
true);
}
}

try {
Expand All @@ -198,7 +221,24 @@ public MmapDefault(CodeContext/*!*/ context, int fileno, long length, string tag
throw;
}
_position = 0L;
}

void CheckFileAccess(MemoryMappedFileAccess mmapAccess, Stream stream) {
bool isValid = mmapAccess switch {
MemoryMappedFileAccess.Read => stream.CanRead,
MemoryMappedFileAccess.ReadWrite => stream.CanRead && stream.CanWrite,
MemoryMappedFileAccess.CopyOnWrite => stream.CanRead,
_ => false
};

if (!isValid) {
if (_handle is not null && _sourceStream is not null) {
_sourceStream.Dispose();
}
throw PythonOps.OSError(PythonExceptions._OSError.ERROR_ACCESS_DENIED, "Invalid access mode");
}
}
} // end of constructor


public object __len__() {
using (new MmapLocker(this)) {
Expand Down Expand Up @@ -325,6 +365,11 @@ private void CloseWorker() {
_view.Flush();
_view.Dispose();
_file.Dispose();
if (_handle is not null) {
// mmap owns _sourceStream too in this case
_sourceStream?.Dispose();
_handle.Dispose();
}
_sourceStream = null;
_view = null;
_file = null;
Expand Down Expand Up @@ -557,6 +602,11 @@ public void resize(long newsize) {
}

if (_sourceStream == null) {
if (_handle is not null && !_handle.IsInvalid
&& (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux))) {
// resize on Posix platforms
PythonNT.ftruncateUnix(unchecked((int)_handle.DangerousGetHandle()), newsize);
}
// resizing is not supported without an underlying file
throw WindowsError(PythonExceptions._OSError.ERROR_INVALID_PARAMETER);
}
Expand Down Expand Up @@ -716,6 +766,9 @@ public void seek(long pos, int whence = SEEK_SET) {

public object size() {
using (new MmapLocker(this)) {
if (_handle is not null && (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux))) {
return GetFileSizeUnix(_handle);
}
if (_sourceStream == null) return ReturnLong(_view.Capacity);
return ReturnLong(new FileInfo(_sourceStream.Name).Length);
}
Expand Down Expand Up @@ -830,6 +883,25 @@ internal Bytes GetSearchString() {
}
}

[SupportedOSPlatform("linux"), SupportedOSPlatform("macos")]
private static long GetFileSizeUnix(SafeFileHandle handle) {
long size;
if (handle.IsInvalid) {
throw PythonOps.OSError(PythonExceptions._OSError.ERROR_INVALID_HANDLE, "Invalid file handle");
}

if (Mono.Unix.Native.Syscall.fstat((int)handle.DangerousGetHandle(), out Mono.Unix.Native.Stat status) == 0) {
size = status.st_size;
} else {
Mono.Unix.Native.Errno errno = Mono.Unix.Native.Stdlib.GetLastError();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can't use the PythonNT.GetLastUnixError here?

Copy link
Member Author

@BCSharp BCSharp Jan 8, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PythonNT.GetLastUnixError cannot be used here (yet) because it does not have proper platform guards (yet). Adding the platform guards to PythonNT.GetLastUnixError has a cascading effect to the code. I am saving it for a separate PR (soon), not to mix up too much with this one.

string msg = Mono.Unix.UnixMarshal.GetErrorDescription(errno);
int error = Mono.Unix.Native.NativeConvert.FromErrno(errno);
throw PythonOps.OSError(error, msg);
}

return size;
}

#endregion

#region Synchronization
Expand Down
Loading
Loading