Repository navigation
Commit 058b141
committed
Py_UniversalNewlineFread(): Many changes.
+ Continued looping until n bytes in the buffer have been filled, not
just when n bytes have been read from the file. This repairs the
bug that f.readlines() only sucked up the first 8192 bytes of the file
on Windows when universal newlines was enabled and f was opened in
U mode (see Python-Dev -- this was the ultimate cause of the
test_inspect.py failure).
+ Changed prototye to take a char* buffer (void* doesn't make much sense).
+ Squashed size_t vs int mismatches (in particular, besides the unsigned
vs signed distinction, size_t may be larger than int).
+ Gets out under all error conditions now (it's possible for fread() to
suffer an error even if it returns a number larger than 0 -- any
"short read" is an error or EOF condition).
+ Rearranged and simplified declarations.1 parent ea572b2 commit 058b141
2 files changed
Lines changed: 52 additions & 42 deletions
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -41,7 +41,7 @@ extern DL_IMPORT(int) PyFile_WriteString(const char *, PyObject *); | |||
| 41 | 41 | extern DL_IMPORT(int) PyObject_AsFileDescriptor(PyObject *); | |
| 42 | 42 | ||
| 43 | 43 | /* The default encoding used by the platform file system APIs | |
| 44 | - If non-NULL, this is different than the default encoding for strings | ||
| 44 | + If non-NULL, this is different than the default encoding for strings | ||
| 45 | 45 | */ | |
| 46 | 46 | extern DL_IMPORT(const char *) Py_FileSystemDefaultEncoding; | |
| 47 | 47 | ||
@@ -51,12 +51,12 @@ extern DL_IMPORT(const char *) Py_FileSystemDefaultEncoding; | |||
| 51 | 51 | */ | |
| 52 | 52 | #define PY_STDIOTEXTMODE "b" | |
| 53 | 53 | char *Py_UniversalNewlineFgets(char *, int, FILE*, PyObject *); | |
| 54 | - size_t Py_UniversalNewlineFread(void *, size_t, FILE *, PyObject *); | ||
| 54 | + size_t Py_UniversalNewlineFread(char *, size_t, FILE *, PyObject *); | ||
| 55 | 55 | #else | |
| 56 | 56 | #define PY_STDIOTEXTMODE "" | |
| 57 | - #define Py_UniversalNewlineFgets(buf, len, fp, obj) (fgets((buf), (len), (fp))) | ||
| 58 | - #define Py_UniversalNewlineFread(buf, len, fp, obj) \ | ||
| 59 | - (fread((buf), 1, (len), (fp))) | ||
| 57 | + #define Py_UniversalNewlineFgets(buf, len, fp, obj) fgets((buf), (len), (fp)) | ||
| 58 | + #define Py_UniversalNewlineFread(buf, len, fp, obj) | ||
| 59 | + fread((buf), 1, (len), (fp)) | ||
| 60 | 60 | #endif /* WITH_UNIVERSAL_NEWLINES */ | |
| 61 | 61 | #ifdef __cplusplus | |
| 62 | 62 | } | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1228,7 +1228,7 @@ file_readlines(PyFileObject *f, PyObject *args) | |||
| 1228 | 1228 | else { | |
| 1229 | 1229 | Py_BEGIN_ALLOW_THREADS | |
| 1230 | 1230 | errno = 0; | |
| 1231 | - nread = Py_UniversalNewlineFread(buffer+nfilled, | ||
| 1231 | + nread = Py_UniversalNewlineFread(buffer+nfilled, | ||
| 1232 | 1232 | buffersize-nfilled, f->f_fp, (PyObject *)f); | |
| 1233 | 1233 | Py_END_ALLOW_THREADS | |
| 1234 | 1234 | shortread = (nread < buffersize-nfilled); | |
@@ -1943,7 +1943,7 @@ Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj) | |||
| 1943 | 1943 | int newlinetypes = 0; | |
| 1944 | 1944 | int skipnextlf = 0; | |
| 1945 | 1945 | int univ_newline = 1; | |
| 1946 | - | ||
| 1946 | + | ||
| 1947 | 1947 | if (fobj) { | |
| 1948 | 1948 | if (!PyFile_Check(fobj)) { | |
| 1949 | 1949 | errno = ENXIO; /* What can you do... */ | |
@@ -2024,61 +2024,71 @@ Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj) | |||
| 2024 | 2024 | ** the different types of newlines seen. | |
| 2025 | 2025 | */ | |
| 2026 | 2026 | size_t | |
| 2027 | - Py_UniversalNewlineFread(void *buf, size_t n, | ||
| 2027 | + Py_UniversalNewlineFread(char *buf, size_t n, | ||
| 2028 | 2028 | FILE *stream, PyObject *fobj) | |
| 2029 | 2029 | { | |
| 2030 | - char *src = buf, *dst = buf, c; | ||
| 2031 | - int nread, ntodo=n; | ||
| 2032 | - int newlinetypes, skipnextlf, univ_newline; | ||
| 2033 | - | ||
| 2030 | + char *dst = buf; | ||
| 2031 | + PyFileObject *f = (PyFileObject *)fobj; | ||
| 2032 | + int newlinetypes, skipnextlf; | ||
| 2033 | + | ||
| 2034 | + assert(buf != NULL); | ||
| 2035 | + assert(stream != NULL); | ||
| 2036 | + | ||
| 2034 | 2037 | if (!fobj || !PyFile_Check(fobj)) { | |
| 2035 | 2038 | errno = ENXIO; /* What can you do... */ | |
| 2036 | 2039 | return -1; | |
| 2037 | 2040 | } | |
| 2038 | - univ_newline = ((PyFileObject *)fobj)->f_univ_newline; | ||
| 2039 | - if ( !univ_newline ) | ||
| 2041 | + if (!f->f_univ_newline) | ||
| 2040 | 2042 | return fread(buf, 1, n, stream); | |
| 2041 | - newlinetypes = ((PyFileObject *)fobj)->f_newlinetypes; | ||
| 2042 | - skipnextlf = ((PyFileObject *)fobj)->f_skipnextlf; | ||
| 2043 | - while (ntodo > 0) { | ||
| 2044 | - if (ferror(stream)) | ||
| 2045 | - break; | ||
| 2046 | - nread = fread(dst, 1, ntodo, stream); | ||
| 2047 | - src = dst; | ||
| 2048 | - if (nread <= 0) { | ||
| 2049 | - if (skipnextlf) | ||
| 2050 | - newlinetypes |= NEWLINE_CR; | ||
| 2051 | - break; | ||
| 2052 | - } | ||
| 2053 | - ntodo -= nread; | ||
| 2054 | - while ( nread-- ) { | ||
| 2055 | - c = *src++; | ||
| 2043 | + newlinetypes = f->f_newlinetypes; | ||
| 2044 | + skipnextlf = f->f_skipnextlf; | ||
| 2045 | + /* Invariant: n is the number of bytes remaining to be filled | ||
| 2046 | + * in the buffer. | ||
| 2047 | + */ | ||
| 2048 | + while (n) { | ||
| 2049 | + size_t nread; | ||
| 2050 | + int shortread; | ||
| 2051 | + char *src = dst; | ||
| 2052 | + | ||
| 2053 | + nread = fread(dst, 1, n, stream); | ||
| 2054 | + assert(nread <= n); | ||
| 2055 | + shortread = nread != n; /* true iff EOF or error */ | ||
| 2056 | + while (nread--) { | ||
| 2057 | + char c = *src++; | ||
| 2056 | 2058 | if (c == '\r') { | |
| 2057 | - /* Save CR as LF and set flag to skip next newline | ||
| 2058 | - */ | ||
| 2059 | + /* Save as LF and set flag to skip next LF. */ | ||
| 2059 | 2060 | *dst++ = '\n'; | |
| 2061 | + --n; | ||
| 2060 | 2062 | skipnextlf = 1; | |
| 2061 | - } else if (skipnextlf && c == '\n') { | ||
| 2062 | - /* Skip an LF, and remember that we saw CR LF | ||
| 2063 | - */ | ||
| 2063 | + } | ||
| 2064 | + else if (skipnextlf && c == '\n') { | ||
| 2065 | + /* Skip LF, and remember we saw CR LF. */ | ||
| 2064 | 2066 | skipnextlf = 0; | |
| 2065 | 2067 | newlinetypes |= NEWLINE_CRLF; | |
| 2066 | - } else { | ||
| 2067 | - /* Normal char to be stored in buffer. Also update | ||
| 2068 | - ** the newlinetypes flag if either this is an LF | ||
| 2069 | - ** or the previous char was a CR. | ||
| 2070 | - */ | ||
| 2068 | + } | ||
| 2069 | + else { | ||
| 2070 | + /* Normal char to be stored in buffer. Also | ||
| 2071 | + * update the newlinetypes flag if either this | ||
| 2072 | + * is an LF or the previous char was a CR. | ||
| 2073 | + */ | ||
| 2071 | 2074 | if (c == '\n') | |
| 2072 | 2075 | newlinetypes |= NEWLINE_LF; | |
| 2073 | 2076 | else if (skipnextlf) | |
| 2074 | 2077 | newlinetypes |= NEWLINE_CR; | |
| 2075 | 2078 | *dst++ = c; | |
| 2079 | + --n; | ||
| 2076 | 2080 | skipnextlf = 0; | |
| 2077 | 2081 | } | |
| 2078 | 2082 | } | |
| 2083 | + if (shortread) { | ||
| 2084 | + /* If this is EOF, update type flags. */ | ||
| 2085 | + if (skipnextlf && feof(stream)) | ||
| 2086 | + newlinetypes |= NEWLINE_CR; | ||
| 2087 | + break; | ||
| 2088 | + } | ||
| 2079 | 2089 | } | |
| 2080 | - ((PyFileObject *)fobj)->f_newlinetypes = newlinetypes; | ||
| 2081 | - ((PyFileObject *)fobj)->f_skipnextlf = skipnextlf; | ||
| 2082 | - return dst - (char *)buf; | ||
| 2090 | + f->f_newlinetypes = newlinetypes; | ||
| 2091 | + f->f_skipnextlf = skipnextlf; | ||
| 2092 | + return dst - buf; | ||
| 2083 | 2093 | } | |
| 2084 | 2094 | #endif | |
0 commit comments