changeset: 94053:89bb32b9ea99
tag: tip
user: Victor Stinner
date: Thu Jan 08 11:00:08 2015 +0100
files: Include/fileutils.h Modules/main.c Modules/mmapmodule.c Modules/posixmodule.c Modules/signalmodule.c Programs/_freeze_importlib.c Python/dynload_shlib.c Python/fileutils.c Python/marshal.c Python/random.c Python/sysmodule.c
description:
Issue #23152: Implement _Py_fstat() to support files larger than 2 GB on
Windows.
fstat() may fail with EOVERFLOW on files larger than 2 GB because the file size
type is an signed 32-bit integer.
diff -r 1bb3df5bb83f -r 89bb32b9ea99 Include/fileutils.h
--- a/Include/fileutils.h Tue Jan 06 14:05:03 2015 +0100
+++ b/Include/fileutils.h Thu Jan 08 11:00:08 2015 +0100
@@ -21,11 +21,40 @@ PyAPI_FUNC(int) _Py_wstat(
struct stat *buf);
#endif
+#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
+
+#ifdef MS_WINDOWS
+typedef struct _Py_stat_struct {
+ unsigned long st_dev;
+ __int64 st_ino;
+ unsigned short st_mode;
+ int st_nlink;
+ int st_uid;
+ int st_gid;
+ unsigned long st_rdev;
+ __int64 st_size;
+ time_t st_atime;
+ int st_atime_nsec;
+ time_t st_mtime;
+ int st_mtime_nsec;
+ time_t st_ctime;
+ int st_ctime_nsec;
+ unsigned long st_file_attributes;
+};
+#else
+# define _Py_stat_struct stat
+#endif
+
+PyAPI_FUNC(int) _Py_fstat(
+ int fd,
+ struct _Py_stat_struct *stat);
+#endif /* HAVE_FSTAT || MS_WINDOWS */
+
#ifdef HAVE_STAT
PyAPI_FUNC(int) _Py_stat(
PyObject *path,
struct stat *statbuf);
-#endif
+#endif /* HAVE_STAT */
#ifndef Py_LIMITED_API
PyAPI_FUNC(int) _Py_open(
diff -r 1bb3df5bb83f -r 89bb32b9ea99 Modules/main.c
--- a/Modules/main.c Tue Jan 06 14:05:03 2015 +0100
+++ b/Modules/main.c Thu Jan 08 11:00:08 2015 +0100
@@ -752,9 +752,8 @@ Py_Main(int argc, wchar_t **argv)
}
}
{
- /* XXX: does this work on Win/Win64? (see posix_fstat) */
- struct stat sb;
- if (fstat(fileno(fp), &sb) == 0 &&
+ struct _Py_stat_struct sb;
+ if (_Py_fstat(fileno(fp), &sb) == 0 &&
S_ISDIR(sb.st_mode)) {
fprintf(stderr, "%ls: '%ls' is a directory, cannot continue\n", argv[0], filename);
fclose(fp);
diff -r 1bb3df5bb83f -r 89bb32b9ea99 Modules/mmapmodule.c
--- a/Modules/mmapmodule.c Tue Jan 06 14:05:03 2015 +0100
+++ b/Modules/mmapmodule.c Thu Jan 08 11:00:08 2015 +0100
@@ -459,8 +459,8 @@ mmap_size_method(mmap_object *self,
#ifdef UNIX
{
- struct stat buf;
- if (-1 == fstat(self->fd, &buf)) {
+ struct _Py_stat_struct buf;
+ if (-1 == _Py_fstat(self->fd, &buf)) {
PyErr_SetFromErrno(PyExc_OSError);
return NULL;
}
@@ -1107,7 +1107,7 @@ static PyObject *
new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
{
#ifdef HAVE_FSTAT
- struct stat st;
+ struct _Py_stat_struct st;
#endif
mmap_object *m_obj;
PyObject *map_size_obj = NULL;
@@ -1174,7 +1174,7 @@ new_mmap_object(PyTypeObject *type, PyOb
(void)fcntl(fd, F_FULLFSYNC);
#endif
#ifdef HAVE_FSTAT
- if (fd != -1 && fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
+ if (fd != -1 && _Py_fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
if (map_size == 0) {
if (st.st_size == 0) {
PyErr_SetString(PyExc_ValueError,
diff -r 1bb3df5bb83f -r 89bb32b9ea99 Modules/posixmodule.c
--- a/Modules/posixmodule.c Tue Jan 06 14:05:03 2015 +0100
+++ b/Modules/posixmodule.c Thu Jan 08 11:00:08 2015 +0100
@@ -350,7 +350,7 @@ static int win32_can_symlink = 0;
#ifdef MS_WINDOWS
# define STAT win32_stat
# define LSTAT win32_lstat
-# define FSTAT win32_fstat
+# define FSTAT _Py_fstat
# define STRUCT_STAT struct win32_stat
#else
# define STAT stat
@@ -1443,73 +1443,6 @@ struct win32_stat{
unsigned long st_file_attributes;
};
-static __int64 secs_between_epochs = 11644473600; /* Seconds between 1.1.1601 and 1.1.1970 */
-
-static void
-FILE_TIME_to_time_t_nsec(FILETIME *in_ptr, time_t *time_out, int* nsec_out)
-{
- /* XXX endianness. Shouldn't matter, as all Windows implementations are little-endian */
- /* Cannot simply cast and dereference in_ptr,
- since it might not be aligned properly */
- __int64 in;
- memcpy(&in, in_ptr, sizeof(in));
- *nsec_out = (int)(in % 10000000) * 100; /* FILETIME is in units of 100 nsec. */
- *time_out = Py_SAFE_DOWNCAST((in / 10000000) - secs_between_epochs, __int64, time_t);
-}
-
-static void
-time_t_to_FILE_TIME(time_t time_in, int nsec_in, FILETIME *out_ptr)
-{
- /* XXX endianness */
- __int64 out;
- out = time_in + secs_between_epochs;
- out = out * 10000000 + nsec_in / 100;
- memcpy(out_ptr, &out, sizeof(out));
-}
-
-/* Below, we *know* that ugo+r is 0444 */
-#if _S_IREAD != 0400
-#error Unsupported C library
-#endif
-static int
-attributes_to_mode(DWORD attr)
-{
- int m = 0;
- if (attr & FILE_ATTRIBUTE_DIRECTORY)
- m |= _S_IFDIR | 0111; /* IFEXEC for user,group,other */
- else
- m |= _S_IFREG;
- if (attr & FILE_ATTRIBUTE_READONLY)
- m |= 0444;
- else
- m |= 0666;
- return m;
-}
-
-static int
-attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, struct win32_stat *result)
-{
- memset(result, 0, sizeof(*result));
- result->st_mode = attributes_to_mode(info->dwFileAttributes);
- result->st_size = (((__int64)info->nFileSizeHigh)st_dev = info->dwVolumeSerialNumber;
- result->st_rdev = result->st_dev;
- FILE_TIME_to_time_t_nsec(&info->ftCreationTime, &result->st_ctime, &result->st_ctime_nsec);
- FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec);
- FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec);
- result->st_nlink = info->nNumberOfLinks;
- result->st_ino = (((__int64)info->nFileIndexHigh)st_mode ^= (result->st_mode & S_IFMT);
- /* now set the bits that make this a symlink */
- result->st_mode |= S_IFLNK;
- }
- result->st_file_attributes = info->dwFileAttributes;
-
- return 0;
-}
-
static BOOL
attributes_from_dir(LPCSTR pszFile, BY_HANDLE_FILE_INFORMATION *info, ULONG *reparse_tag)
{
@@ -1870,57 +1803,6 @@ win32_stat_w(const wchar_t* path, struct
return win32_xstat_w(path, result, TRUE);
}
-static int
-win32_fstat(int file_number, struct win32_stat *result)
-{
- BY_HANDLE_FILE_INFORMATION info;
- HANDLE h;
- int type;
-
- if (!_PyVerify_fd(file_number))
- h = INVALID_HANDLE_VALUE;
- else
- h = (HANDLE)_get_osfhandle(file_number);
-
- /* Protocol violation: we explicitly clear errno, instead of
- setting it to a POSIX error. Callers should use GetLastError. */
- errno = 0;
-
- if (h == INVALID_HANDLE_VALUE) {
- /* This is really a C library error (invalid file handle).
- We set the Win32 error to the closes one matching. */
- SetLastError(ERROR_INVALID_HANDLE);
- return -1;
- }
- memset(result, 0, sizeof(*result));
-
- type = GetFileType(h);
- if (type == FILE_TYPE_UNKNOWN) {
- DWORD error = GetLastError();
- if (error != 0) {
- return -1;
- }
- /* else: valid but unknown file */
- }
-
- if (type != FILE_TYPE_DISK) {
- if (type == FILE_TYPE_CHAR)
- result->st_mode = _S_IFCHR;
- else if (type == FILE_TYPE_PIPE)
- result->st_mode = _S_IFIFO;
- return 0;
- }
-
- if (!GetFileInformationByHandle(h, &info)) {
- return -1;
- }
-
- attribute_data_to_stat(&info, 0, result);
- /* specific to fstat() */
- result->st_ino = (((__int64)info.nFileIndexHigh)