FazBrowse GitHub Viewer
|
Trending
|
URL:
|
Home
Tools:
[Download Repo ZIP]
[View Raw Code]
[Original HTTPS Page]
scrapy/tests/test_command_shell.py at master · jackcoolpython/scrapy · GitHub
jackcoolpython
scrapy
Repository navigation
Code
Pull requests
Actions
Projects
Security and quality
Insights
Expand file tree
Breadcrumbs
scrapy
/
tests
/
test_command_shell.py
Copy path
More file actions
More file actions
Latest commit
History
History
History
115 lines (93 loc) · 4.3 KB
Breadcrumbs
scrapy
/
tests
/
test_command_shell.py
Copy path
File metadata and controls
115 lines (93 loc) · 4.3 KB
Raw
Copy raw file
Download raw file
Open symbols panel
Edit and raw actions
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
from
os
.
path
import
join
from
twisted
.
trial
import
unittest
from
twisted
.
internet
import
defer
from
scrapy
.
utils
.
testsite
import
SiteTest
from
scrapy
.
utils
.
testproc
import
ProcessTest
from
tests
import
tests_datadir
class
ShellTest
(
ProcessTest
,
SiteTest
,
unittest
.
TestCase
):
command
=
'shell'
@
defer
.
inlineCallbacks
def
test_empty
(
self
):
_
,
out
,
_
=
yield
self
.
execute
([
'-c'
,
'item'
])
assert
b'{}'
in
out
@
defer
.
inlineCallbacks
def
test_response_body
(
self
):
_
,
out
,
_
=
yield
self
.
execute
([
self
.
url
(
'/text'
),
'-c'
,
'response.body'
])
assert
b'Works'
in
out
@
defer
.
inlineCallbacks
def
test_response_type_text
(
self
):
_
,
out
,
_
=
yield
self
.
execute
([
self
.
url
(
'/text'
),
'-c'
,
'type(response)'
])
assert
b'TextResponse'
in
out
@
defer
.
inlineCallbacks
def
test_response_type_html
(
self
):
_
,
out
,
_
=
yield
self
.
execute
([
self
.
url
(
'/html'
),
'-c'
,
'type(response)'
])
assert
b'HtmlResponse'
in
out
@
defer
.
inlineCallbacks
def
test_response_selector_html
(
self
):
xpath
=
'response.xpath("//p[@class=
\'
one
\'
]/text()").get()'
_
,
out
,
_
=
yield
self
.
execute
([
self
.
url
(
'/html'
),
'-c'
,
xpath
])
self
.
assertEqual
(
out
.
strip
(),
b'Works'
)
@
defer
.
inlineCallbacks
def
test_response_encoding_gb18030
(
self
):
_
,
out
,
_
=
yield
self
.
execute
([
self
.
url
(
'/enc-gb18030'
),
'-c'
,
'response.encoding'
])
self
.
assertEqual
(
out
.
strip
(),
b'gb18030'
)
@
defer
.
inlineCallbacks
def
test_redirect
(
self
):
_
,
out
,
_
=
yield
self
.
execute
([
self
.
url
(
'/redirect'
),
'-c'
,
'response.url'
])
assert
out
.
strip
().
endswith
(
b'/redirected'
)
@
defer
.
inlineCallbacks
def
test_redirect_follow_302
(
self
):
_
,
out
,
_
=
yield
self
.
execute
([
self
.
url
(
'/redirect-no-meta-refresh'
),
'-c'
,
'response.status'
])
assert
out
.
strip
().
endswith
(
b'200'
)
@
defer
.
inlineCallbacks
def
test_redirect_not_follow_302
(
self
):
_
,
out
,
_
=
yield
self
.
execute
(
[
'--no-redirect'
,
self
.
url
(
'/redirect-no-meta-refresh'
),
'-c'
,
'response.status'
]
)
assert
out
.
strip
().
endswith
(
b'302'
)
@
defer
.
inlineCallbacks
def
test_fetch_redirect_follow_302
(
self
):
"""Test that calling ``fetch(url)`` follows HTTP redirects by default."""
url
=
self
.
url
(
'/redirect-no-meta-refresh'
)
code
=
"fetch('{0}')"
errcode
,
out
,
errout
=
yield
self
.
execute
([
'-c'
,
code
.
format
(
url
)])
self
.
assertEqual
(
errcode
,
0
,
out
)
assert
b'Redirecting (302)'
in
errout
assert
b'Crawled (200)'
in
errout
@
defer
.
inlineCallbacks
def
test_fetch_redirect_not_follow_302
(
self
):
"""Test that calling ``fetch(url, redirect=False)`` disables automatic redirects."""
url
=
self
.
url
(
'/redirect-no-meta-refresh'
)
code
=
"fetch('{0}', redirect=False)"
errcode
,
out
,
errout
=
yield
self
.
execute
([
'-c'
,
code
.
format
(
url
)])
self
.
assertEqual
(
errcode
,
0
,
out
)
assert
b'Crawled (302)'
in
errout
@
defer
.
inlineCallbacks
def
test_request_replace
(
self
):
url
=
self
.
url
(
'/text'
)
code
=
"fetch('{0}') or fetch(response.request.replace(method='POST'))"
errcode
,
out
,
_
=
yield
self
.
execute
([
'-c'
,
code
.
format
(
url
)])
self
.
assertEqual
(
errcode
,
0
,
out
)
@
defer
.
inlineCallbacks
def
test_scrapy_import
(
self
):
url
=
self
.
url
(
'/text'
)
code
=
"fetch(scrapy.Request('{0}'))"
errcode
,
out
,
_
=
yield
self
.
execute
([
'-c'
,
code
.
format
(
url
)])
self
.
assertEqual
(
errcode
,
0
,
out
)
@
defer
.
inlineCallbacks
def
test_local_file
(
self
):
filepath
=
join
(
tests_datadir
,
'test_site/index.html'
)
_
,
out
,
_
=
yield
self
.
execute
([
filepath
,
'-c'
,
'item'
])
assert
b'{}'
in
out
@
defer
.
inlineCallbacks
def
test_local_nofile
(
self
):
filepath
=
'file:///tests/sample_data/test_site/nothinghere.html'
errcode
,
out
,
err
=
yield
self
.
execute
([
filepath
,
'-c'
,
'item'
],
check_code
=
False
)
self
.
assertEqual
(
errcode
,
1
,
out
or
err
)
self
.
assertIn
(
b'No such file or directory'
,
err
)
@
defer
.
inlineCallbacks
def
test_dns_failures
(
self
):
url
=
'www.somedomainthatdoesntexi.st'
errcode
,
out
,
err
=
yield
self
.
execute
([
url
,
'-c'
,
'item'
],
check_code
=
False
)
self
.
assertEqual
(
errcode
,
1
,
out
or
err
)
self
.
assertIn
(
b'DNS lookup failed'
,
err
)
Back
|
FazBrowse Home
|
New Git URL