FazBrowse GitHub Viewer
|
Trending
|
URL:
|
Home
Tools:
[Download Repo ZIP]
[View Raw Code]
[Original HTTPS Page]
codeql-action/src/database-upload.ts at default-setup-testing · github/codeql-action · GitHub
github
codeql-action
Repository navigation
Code
Issues
165
(165)
Pull requests
44
(44)
Actions
Security and quality
2
(2)
Insights
Expand file tree
Breadcrumbs
codeql-action
/
src
/
database-upload.ts
Copy path
More file actions
More file actions
Latest commit
History
History
History
312 lines (291 loc) · 11.4 KB
Breadcrumbs
codeql-action
/
src
/
database-upload.ts
Copy path
File metadata and controls
312 lines (291 loc) · 11.4 KB
Raw
Copy raw file
Download raw file
Open symbols panel
Edit and raw actions
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
import
*
as
fs
from
"fs"
;
import
{
ActionState
}
from
"./action-common"
;
import
*
as
actionsUtil
from
"./actions-util"
;
import
{
AnalysisKind
}
from
"./analyses"
;
import
{
DO_NOT_RETRY_STATUSES
,
getApiClient
,
GitHubApiDetails
,
}
from
"./api-client"
;
import
{
type
CodeQL
}
from
"./codeql"
;
import
{
Config
}
from
"./config-utils"
;
import
{
Feature
}
from
"./feature-flags"
;
import
*
as
gitUtils
from
"./git-utils"
;
import
{
parseGitHubUrl
}
from
"./github"
;
import
{
Logger
,
withGroupAsync
}
from
"./logging"
;
import
{
OverlayDatabaseMode
}
from
"./overlay/overlay-database-mode"
;
import
{
RepositoryNwo
}
from
"./repository"
;
import
*
as
util
from
"./util"
;
import
{
asHTTPError
,
bundleDb
,
CleanupLevel
}
from
"./util"
;
/** Information about a database upload. */
export
interface
DatabaseUploadResult
{
/** Language of the database. */
language
:
string
;
/** Size of the zipped database in bytes. */
zipped_upload_size_bytes
?:
number
;
/** Whether the uploaded database is an overlay base. */
is_overlay_base
?:
boolean
;
/**
* For overlay-base uploads only: the size in bytes that the zipped database
* would have been if it had been cleaned at the `clear` cleanup level instead
* of the `overlay` level.
*/
clear_cleanup_zipped_size_bytes
?:
number
;
/**
* For overlay-base uploads only: the time in milliseconds spent measuring the
* `clear` cleanup size (cleaning up the cluster at the `clear` level and
* bundling each database). This is a cluster-wide measurement, so it is the
* same for every language in a run.
*/
clear_cleanup_measurement_duration_ms
?:
number
;
/** Time taken to upload database in milliseconds. */
upload_duration_ms
?:
number
;
/** If there was an error during database upload, this is its message. */
error
?:
string
;
}
export
async
function
cleanupAndUploadDatabases
(
action
:
ActionState
<
[
"ReadOnlyEnv"
,
"Logger"
,
"FeatureFlags"
]
>
,
repositoryNwo
:
RepositoryNwo
,
codeql
:
CodeQL
,
config
:
Config
,
apiDetails
:
GitHubApiDetails
,
checkoutPath
:
string
,
)
:
Promise
<
DatabaseUploadResult
[
]
>
{
const
logger
=
action
.
logger
;
if
(
actionsUtil
.
getRequiredInput
(
"upload-database"
)
!==
"true"
)
{
logger
.
debug
(
"Database upload disabled in workflow. Skipping upload."
)
;
return
[
]
;
}
if
(
!
config
.
analysisKinds
.
includes
(
AnalysisKind
.
CodeScanning
)
)
{
logger
.
debug
(
`Not uploading database because 'analysis-kinds:
${
AnalysisKind
.
CodeScanning
}
' is not enabled.`
,
)
;
return
[
]
;
}
if
(
util
.
isInTestMode
(
)
)
{
logger
.
debug
(
"In test mode. Skipping database upload."
)
;
return
[
]
;
}
// Do nothing when not running against github.com
if
(
config
.
gitHubVersion
.
type
!==
util
.
GitHubVariant
.
DOTCOM
&&
config
.
gitHubVersion
.
type
!==
util
.
GitHubVariant
.
GHEC_DR
)
{
logger
.
debug
(
"Not running against github.com or GHEC-DR. Skipping upload."
)
;
return
[
]
;
}
if
(
!
(
await
gitUtils
.
isAnalyzingDefaultBranch
(
action
.
env
,
checkoutPath
)
)
)
{
// We only want to upload a database if we are analyzing the default branch.
logger
.
debug
(
"Not analyzing default branch. Skipping upload."
)
;
return
[
]
;
}
// If config.overlayDatabaseMode is OverlayBase, then we have overlay base databases for all languages.
const
shouldUploadOverlayBase
=
config
.
overlayDatabaseMode
===
OverlayDatabaseMode
.
OverlayBase
&&
(
await
action
.
features
.
getValue
(
Feature
.
UploadOverlayDbToApi
,
codeql
)
)
;
const
cleanupLevel
=
shouldUploadOverlayBase
?
CleanupLevel
.
Overlay
:
CleanupLevel
.
Clear
;
// Clean up the database, since intermediate results may still be written to the
// database if there is high RAM pressure.
await
withGroupAsync
(
"Cleaning up databases"
,
async
(
)
=>
{
await
codeql
.
databaseCleanupCluster
(
config
,
cleanupLevel
)
;
}
)
;
const
reports
:
DatabaseUploadResult
[
]
=
[
]
;
for
(
const
language
of
config
.
languages
)
{
let
bundledDbSize
:
number
|
undefined
=
undefined
;
try
{
// Upload the database bundle.
// Although we are uploading arbitrary file contents to the API, it's worth
// noting that it's the API's job to validate that the contents is acceptable.
// This API method is available to anyone with write access to the repo.
const
bundledDb
=
await
bundleDb
(
config
,
language
,
codeql
,
language
,
{
includeDiagnostics
:
false
,
}
)
;
bundledDbSize
=
fs
.
statSync
(
bundledDb
)
.
size
;
const
commitOid
=
await
gitUtils
.
getCommitOid
(
action
.
env
,
checkoutPath
)
;
// Upload with manual retry logic. We disable Octokit's built-in retries
// because the request body is a ReadStream, which can only be consumed
// once.
const
maxAttempts
=
4
;
// 1 initial attempt + 3 retries, identical to the default retry behavior of Octokit
let
uploadDurationMs
:
number
|
undefined
;
for
(
let
attempt
=
1
;
attempt
<=
maxAttempts
;
attempt
++
)
{
try
{
uploadDurationMs
=
await
uploadBundledDatabase
(
repositoryNwo
,
language
,
commitOid
,
bundledDb
,
bundledDbSize
,
apiDetails
,
)
;
break
;
}
catch
(
e
)
{
const
httpError
=
asHTTPError
(
e
)
;
const
isRetryable
=
!
httpError
||
!
DO_NOT_RETRY_STATUSES
.
includes
(
httpError
.
status
)
;
if
(
!
isRetryable
)
{
throw
e
;
}
else
if
(
attempt
===
maxAttempts
)
{
logger
.
error
(
`Maximum retry attempts exhausted (
${
attempt
}
), aborting database upload`
,
)
;
throw
e
;
}
const
backoffMs
=
15_000
*
Math
.
pow
(
2
,
attempt
-
1
)
;
// 15s, 30s, 60s
logger
.
debug
(
`Database upload attempt
${
attempt
}
of
${
maxAttempts
}
failed for
${
language
}
:
${
util
.
getErrorMessage
(
e
)
}
. Retrying in
${
backoffMs
/
1000
}
s...`
,
)
;
await
new
Promise
(
(
resolve
)
=>
setTimeout
(
resolve
,
backoffMs
)
)
;
}
}
reports
.
push
(
{
language
,
zipped_upload_size_bytes
:
bundledDbSize
,
is_overlay_base
:
shouldUploadOverlayBase
,
upload_duration_ms
:
uploadDurationMs
,
}
)
;
logger
.
debug
(
`Successfully uploaded database for
${
language
}
`
)
;
}
catch
(
e
)
{
// Log a warning but don't fail the workflow
logger
.
warning
(
`Failed to upload database for
${
language
}
:
${
util
.
getErrorMessage
(
e
)
}
`
,
)
;
reports
.
push
(
{
language
,
error
:
util
.
getErrorMessage
(
e
)
,
...
(
bundledDbSize
!==
undefined
?
{
zipped_upload_size_bytes
:
bundledDbSize
}
:
{
}
)
,
}
)
;
}
}
// When we upload an overlay-base database, we cleaned the databases at the `overlay` level, which
// retains more data than the `clear` level used for regular uploads. Measure what the zipped size
// would have been at the `clear` level too, so we can compare the storage cost of overlay-base
// databases against regular databases for the same repository.
//
// We skip this in debug mode, where the databases are preserved and uploaded as debug artifacts,
// since cleaning them up at the `clear` level would discard data that is useful for debugging.
if
(
shouldUploadOverlayBase
&&
!
config
.
debugMode
)
{
await
withGroupAsync
(
"Measuring database size at the clear cleanup level"
,
(
)
=>
recordClearCleanupSizes
(
codeql
,
config
,
reports
,
logger
)
,
)
;
}
return
reports
;
}
/**
* Cleans up the databases at the `clear` cleanup level and records the resulting zipped size for
* each language in `clear_cleanup_zipped_size_bytes`. If the cleanup succeeds, also records the
* time spent taking the measurement in `clear_cleanup_measurement_duration_ms`.
*
* This mutates the entries of `reports` in place. It must run only after all overlay-base uploads
* have completed, since the `clear` cleanup discards overlay data that the uploaded database
* depends on.
*
* Failures here are non-fatal: this is telemetry-only, so we log and move on rather than failing
* the workflow.
*/
async
function
recordClearCleanupSizes
(
codeql
:
CodeQL
,
config
:
Config
,
reports
:
DatabaseUploadResult
[
]
,
logger
:
Logger
,
)
:
Promise
<
void
>
{
// Include both the cleanup and the re-bundling to record how much time taking this measurement adds
// to the run.
const
startTime
=
performance
.
now
(
)
;
try
{
await
codeql
.
databaseCleanupCluster
(
config
,
CleanupLevel
.
Clear
)
;
}
catch
(
e
)
{
// The cleanup didn't run, so there are no sizes to measure. Return without recording a
// duration, so that we don't report a measurement duration with no accompanying sizes.
logger
.
warning
(
`Failed to clean up databases at the '
${
CleanupLevel
.
Clear
}
' level for `
+
`size measurement:
${
util
.
getErrorMessage
(
e
)
}
`
,
)
;
return
;
}
for
(
const
language
of
config
.
languages
)
{
const
report
=
reports
.
find
(
(
r
)
=>
r
.
language
===
language
)
;
if
(
report
===
undefined
)
{
continue
;
}
try
{
const
bundledDb
=
await
bundleDb
(
config
,
language
,
codeql
,
language
,
{
includeDiagnostics
:
false
,
}
)
;
report
.
clear_cleanup_zipped_size_bytes
=
fs
.
statSync
(
bundledDb
)
.
size
;
logger
.
debug
(
`Database for
${
language
}
is `
+
`
${
report
.
clear_cleanup_zipped_size_bytes
}
bytes zipped at the `
+
`'
${
CleanupLevel
.
Clear
}
' cleanup level `
+
`(vs.
${
report
.
zipped_upload_size_bytes
??
"unknown"
}
bytes at the `
+
`'
${
CleanupLevel
.
Overlay
}
' level).`
,
)
;
}
catch
(
e
)
{
logger
.
warning
(
`Failed to measure the '
${
CleanupLevel
.
Clear
}
' cleanup database size `
+
`for
${
language
}
:
${
util
.
getErrorMessage
(
e
)
}
`
,
)
;
}
}
const
durationMs
=
performance
.
now
(
)
-
startTime
;
for
(
const
report
of
reports
)
{
report
.
clear_cleanup_measurement_duration_ms
=
durationMs
;
}
}
/**
* Uploads a bundled database to the GitHub API.
*
*
@returns
the duration of the upload in milliseconds
*/
export
async
function
uploadBundledDatabase
(
repositoryNwo
:
RepositoryNwo
,
language
:
string
,
commitOid
:
string
,
bundledDb
:
string
,
bundledDbSize
:
number
,
apiDetails
:
GitHubApiDetails
,
)
:
Promise
<
number
>
{
// Ensure that the `apiDetails` that `getApiClient` uses match the ones we were given here,
// because we use `apiDetails.auth` below which must be used with the same instance that
// `apiDetails` refers to.
const
client
=
getApiClient
(
util
.
getEnv
(
)
,
apiDetails
)
;
const
uploadsUrl
=
new
URL
(
parseGitHubUrl
(
apiDetails
.
url
)
)
;
uploadsUrl
.
hostname
=
`uploads.
${
uploadsUrl
.
hostname
}
`
;
// Octokit expects the baseUrl to not have a trailing slash,
// but it is included by default in a URL.
let
uploadsBaseUrl
=
uploadsUrl
.
toString
(
)
;
if
(
uploadsBaseUrl
.
endsWith
(
"/"
)
)
{
uploadsBaseUrl
=
uploadsBaseUrl
.
slice
(
0
,
-
1
)
;
}
const
bundledDbReadStream
=
fs
.
createReadStream
(
bundledDb
)
;
try
{
const
startTime
=
performance
.
now
(
)
;
await
client
.
request
(
`POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`
,
{
baseUrl
:
uploadsBaseUrl
,
owner
:
repositoryNwo
.
owner
,
repo
:
repositoryNwo
.
repo
,
language
,
name
:
`
${
language
}
-database`
,
commit_oid
:
commitOid
,
data
:
bundledDbReadStream
,
headers
:
{
"Content-Type"
:
"application/zip"
,
"Content-Length"
:
bundledDbSize
,
}
,
// Disable `octokit/plugin-retry.js`, since the request body is a ReadStream which can only be consumed once.
request
:
{
retries
:
0
,
}
,
}
,
)
;
return
performance
.
now
(
)
-
startTime
;
}
finally
{
bundledDbReadStream
.
close
(
)
;
}
}
Back
|
FazBrowse Home
|
New Git URL