FazBrowse GitHub Viewer
|
Trending
|
URL:
|
Home
Tools:
[Download Repo ZIP]
[View Raw Code]
[Original HTTPS Page]
lambda-scraper/lambda/proxy.js at main · git-moeen/lambda-scraper · GitHub
git-moeen
/
lambda-scraper
Public
forked from
teticio/lambda-scraper
Notifications
You must be signed in to change notification settings
Fork
0
Star
0
Code
Pull requests
0
Actions
Projects
Security and quality
0
Insights
Additional navigation options
Code
Pull requests
Actions
Projects
Security and quality
Insights
Expand file tree
Breadcrumbs
lambda-scraper
/
lambda
/
proxy.js
Copy path
More file actions
More file actions
Latest commit
History
History
History
107 lines (99 loc) · 3.93 KB
Breadcrumbs
lambda-scraper
/
lambda
/
proxy.js
Copy path
File metadata and controls
107 lines (99 loc) · 3.93 KB
Raw
Copy raw file
Download raw file
Open symbols panel
Edit and raw actions
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
if
(
typeof
(
awslambda
)
===
'undefined'
)
{
// For testing
global
.
awslambda
=
require
(
'./awslambda'
)
;
}
const
axios
=
require
(
'axios'
)
;
const
pipeline
=
require
(
'util'
)
.
promisify
(
require
(
'stream'
)
.
pipeline
)
;
const
{
defaultProvider
}
=
require
(
"@aws-sdk/credential-provider-node"
)
;
const
{
SignatureV4
}
=
require
(
"@smithy/signature-v4"
)
;
const
{
Sha256
}
=
require
(
"@aws-crypto/sha256-js"
)
;
const
proxyUrls
=
require
(
'./proxy-urls.json'
)
;
exports
.
lambdaHandler
=
awslambda
.
streamifyResponse
(
async
(
event
,
responseStream
,
context
)
=>
{
try
{
const
random
=
Math
.
floor
(
Math
.
random
(
)
*
proxyUrls
.
length
)
;
const
proxyUrl
=
new
URL
(
proxyUrls
[
random
]
)
;
console
.
log
(
'proxy-'
+
String
(
random
)
+
': '
+
proxyUrl
)
;
let
headers
=
Object
.
fromEntries
(
Object
.
entries
(
event
.
headers
)
.
filter
(
(
[
key
]
)
=>
!
key
.
toLowerCase
(
)
.
startsWith
(
'x-amz'
)
&&
!
key
.
toLowerCase
(
)
.
startsWith
(
'x-forwarded-'
)
&&
key
.
toLowerCase
(
)
!==
'host'
)
.
map
(
(
[
key
,
value
]
)
=>
[
key
.
toLowerCase
(
)
===
'authorization'
?
'lambda-scraper-'
+
key
:
key
,
value
]
)
)
;
headers
[
'host'
]
=
proxyUrl
.
hostname
;
// URL encoding and decoding can be ambiguous (e.g. type=a&type=b becomes type=a,b)
// Signing requires the query parameters to be encoded so we pass the raw query string in a header instead
headers
[
'lambda-scraper-raw-query-string'
]
=
event
.
rawQueryString
;
const
httpRequest
=
{
method
:
event
.
requestContext
.
http
.
method
,
path
:
event
.
rawPath
,
// Needed for SignatureV4
url
:
proxyUrl
+
event
.
rawPath
.
substring
(
1
)
,
// query: event.queryStringParameters, // Needed for SignatureV4
// params: event.queryStringParameters,
headers
:
headers
,
responseType
:
'stream'
,
timeout
:
600
*
1000
,
// 10 minutes
}
;
if
(
event
.
body
)
{
httpRequest
.
body
=
event
.
body
;
// Needed for SignatureV4
httpRequest
.
data
=
event
.
body
;
}
const
credentials
=
await
defaultProvider
(
)
(
)
;
const
signer
=
new
SignatureV4
(
{
credentials
:
credentials
,
region
:
process
.
env
.
AWS_REGION
,
service
:
'lambda'
,
sha256
:
Sha256
,
}
)
;
const
signedRequest
=
await
signer
.
sign
(
httpRequest
)
;
let
httpResponse
;
try
{
httpResponse
=
await
axios
(
signedRequest
)
;
}
catch
(
error
)
{
if
(
error
.
response
)
{
await
pipeline
(
error
.
response
.
data
,
awslambda
.
HttpResponseStream
.
from
(
responseStream
,
{
statusCode
:
error
.
response
.
status
,
headers
:
error
.
response
.
headers
,
}
)
,
)
;
return
;
}
throw
error
;
}
headers
=
Object
.
fromEntries
(
Object
.
entries
(
httpResponse
.
headers
)
.
filter
(
(
[
key
]
)
=>
!
key
.
toLowerCase
(
)
.
startsWith
(
'x-amz'
)
)
)
;
await
pipeline
(
httpResponse
.
data
,
awslambda
.
HttpResponseStream
.
from
(
responseStream
,
{
statusCode
:
httpResponse
.
status
,
headers
:
headers
,
}
)
,
)
;
}
catch
(
error
)
{
console
.
error
(
error
)
;
responseStream
=
awslambda
.
HttpResponseStream
.
from
(
responseStream
,
{
statusCode
:
500
,
headers
:
{
'Content-Type'
:
'application/json'
,
}
}
)
;
responseStream
.
write
(
JSON
.
stringify
(
{
message
:
error
.
message
,
stack
:
error
.
stack
}
)
)
;
responseStream
.
end
(
)
;
}
}
)
;
if
(
require
.
main
===
module
)
{
// For testing
const
event
=
{
rawPath
:
'/https://ipinfo.io/ip'
,
rawQueryString
:
''
,
headers
:
{
}
,
requestContext
:
{
http
:
{
method
:
'GET'
,
}
}
}
;
exports
.
lambdaHandler
(
event
,
{
}
)
.
then
(
)
.
catch
(
error
=>
console
.
error
(
error
)
)
;
}
Back
|
FazBrowse Home
|
New Git URL