FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

feat: add partial manifest by olblak · Pull Request #5508 · updatecli/updatecli · GitHub

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension .go  (5) .yaml  (5) All 2 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
8 changes: 8 additions & 0 deletions e2e/updatecli.d/success.d/partial/_source.yaml
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
sources:
getLatestJenkinsStable:
kind: jenkins
getLatestJenkinsWeekly:
kind: jenkins
spec:
release: weekly

30 changes: 30 additions & 0 deletions e2e/updatecli.d/success.d/partial/jenkins.yaml
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
name: "Bump Jenkins weekly version"
pipelineid: "e2e/jenkins"

conditions:
checkIfThereIsSpecificReleaseVersion:
name: Check if 2.263.3 exists as a stable jenkins release
kind: jenkins
disablesourceinput: true
spec:
version: "2.263.3"
checkIfSourceIsLatestStable:
kind: jenkins
name: Check if the source value from 'getLatestJenkinsStable' is the latest Jenkins stable version
sourceid: getLatestJenkinsStable
spec:
release: stable
checkIfThereIsSpecificWeeklyVersion:
name: Check if 2.231 exists as a weekly jenkins release
kind: jenkins
disablesourceinput: true
spec:
version: "2.231"
release: weekly
checkIfSourceIsLatestWeekly:
kind: jenkins
name: Check if the source value from 'getLatestJenkinsWeekly' is the latest Jenkins weekly version
sourceid: getLatestJenkinsWeekly
spec:
release: weekly

This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
sources:
getLatestJenkinsStable:
kind: jenkins
getLatestJenkinsWeekly:
kind: jenkins
spec:
release: stable

42 changes: 34 additions & 8 deletions pkg/core/config/main.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,9 @@ type Spec struct {
type Option struct {
// ManifestFile contains the updatecli manifest full file path
ManifestFile string
// PartialFiles contains the list of full file paths for updatecli partial manifests.
// A partial file is a default manifest snippet available to all manifests within the same directory.
PartialFiles []string
// ValuesFiles contains the list of updatecli values full file path
ValuesFiles []string
// SecretsFiles contains the list of updatecli sops secrets full file path
Expand Down Expand Up @@ -220,23 +223,46 @@ func New(option Option) (configs []Config, err error) {
return configs, err
}

logrus.Infof("Loading Pipeline %q", option.ManifestFile)
readFile := func(path string) ([]byte, error) {
// Load updatecli manifest no matter the file extension
f, err := os.Open(path)

// Load updatecli manifest no matter the file extension
c, err := os.Open(option.ManifestFile)
if err != nil {
return nil, err
}

if err != nil {
return configs, err
return io.ReadAll(f)
}

defer c.Close()

var templatedManifestContent []byte
rawManifestContent, err := io.ReadAll(c)
var rawManifestContent []byte

for _, partialFile := range option.PartialFiles {
partialContent, err := readFile(partialFile)
if err != nil {
return nil, fmt.Errorf("loading Updatecli partial manifest %q: %v", partialFile, err)
}

// Ignore partial files that are not in the same directory as the main manifest file
// This is to avoid loading partial files from other directories to reduce the complexity of the manifest.
if filepath.Dir(partialFile) != filepath.Dir(option.ManifestFile) {
logrus.Debugf("Ignoring partial from a different directory: %q", partialFile)
continue
}

logrus.Debugf("Partial content detected from: %q", partialFile)
rawManifestContent = append(rawManifestContent, partialContent...)
}

logrus.Infof("Loading Pipeline %q", option.ManifestFile)
// Load updatecli manifest no matter the file extension
rawManifestFileContent, err := readFile(option.ManifestFile)
if err != nil {
return configs, err
}

rawManifestContent = append(rawManifestContent, rawManifestFileContent...)
Comment thread
olblak marked this conversation as resolved.

specs := []Spec{}

isCue := false
Expand Down
29 changes: 24 additions & 5 deletions pkg/core/engine/configuration.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,26 @@ func (e *Engine) LoadConfigurations() error {
}
}

for _, manifestFile := range sanitizeUpdatecliManifestFilePath(e.Options.Manifests[i].Manifests) {
manifestFiles, manifestPartials := sanitizeUpdatecliManifestFilePath(e.Options.Manifests[i].Manifests)
for _, manifestFile := range manifestFiles {
var err error

formatErr := func() {
switch len(manifestPartials) {
case 0:
err = fmt.Errorf("%s:\n%s", manifestFile, err)
default:
err = fmt.Errorf("%s:\n* Partial files:\n\t* %s\n* Error:\n\t%s",
manifestFile,
strings.Join(manifestPartials, "\n\t* "),
strings.ReplaceAll(err.Error(), "\n", "\n\t"),
)
}
}

loadedConfigurations, err := config.New(
config.Option{
PartialFiles: manifestPartials,
ManifestFile: manifestFile,
SecretsFiles: e.Options.Manifests[i].Secrets,
ValuesFiles: e.Options.Manifests[i].Values,
Expand All @@ -75,7 +91,9 @@ func (e *Engine) LoadConfigurations() error {
case nil:
// nothing to do
default:
err = fmt.Errorf("%q - %s", manifestFile, err)

formatErr()

errs = append(errs, err)
e.Reports = append(e.Reports,
reports.Report{
Expand All @@ -98,8 +116,9 @@ func (e *Engine) LoadConfigurations() error {
e.Pipelines = append(e.Pipelines, &newPipeline)
e.configurations = append(e.configurations, &loadedConfiguration)
} else {
// don't initially fail as init. of the pipeline still fails even with a successful validation
err := fmt.Errorf("%q - %s", manifestFile, err)

formatErr()

errs = append(errs, err)
e.Reports = append(e.Reports,
reports.Report{
Expand All @@ -121,7 +140,7 @@ func (e *Engine) LoadConfigurations() error {
e := errors.New("failed loading pipeline(s)")

for _, err := range errs {
e = fmt.Errorf("%s\n\t* %s", e.Error(), strings.ReplaceAll(err.Error(), "\n", "\n\t\t* "))
e = fmt.Errorf("%s\n\t* %s", e.Error(), strings.ReplaceAll(err.Error(), "\n", "\n\t\t"))
if errors.Is(err, ErrNoManifestDetected) {
return err
}
Expand Down
16 changes: 15 additions & 1 deletion pkg/core/engine/configuration_test.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,18 @@ func TestLoadConfigurations(t *testing.T) {
},
expectedPipelines: 1,
},
{
name: "Success - Partial with one manifest",
wd: "testdata/partialOneManifest",
engine: Engine{
Options: Options{
Manifests: []manifest.Manifest{
{},
},
},
},
expectedPipelines: 1,
},
{
name: "Success - Default manifest directory",
wd: "testdata/defaultManifestDirname_single",
Expand Down Expand Up @@ -67,7 +79,9 @@ func TestLoadConfigurations(t *testing.T) {
expectedPipelines: 1,
expectedReports: 1,
wantErr: true,
expectedError: `"updatecli.d/failure.yaml" - scm ID "updatecli" from source ID "adopters" doesn't exist`,
expectedError: `failed loading pipeline(s)
* updatecli.d/failure.yaml:
scm ID "updatecli" from source ID "adopters" doesn't exist`,
},
}
for _, tt := range tests {
Expand Down
5 changes: 4 additions & 1 deletion pkg/core/engine/registry.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,10 @@ func (e *Engine) PushToRegistry(manifests, valuesFiles, secretsFiles, policyRefe

joinWithFileStore(manifests)

manifests = sanitizeUpdatecliManifestFilePath(manifests)
manifestFiles, partialFiles := sanitizeUpdatecliManifestFilePath(manifests)

manifests = append(manifests, manifestFiles...)
manifests = append(manifests, partialFiles...)

relativeFromFileStore(manifests)

Expand Down
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
sources:
jenkins:
name: Get Jenkins version
kind: jenkins
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
name: Update Jenkins version
80 changes: 60 additions & 20 deletions pkg/core/engine/utils.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -10,39 +10,79 @@ import (
"github.com/updatecli/updatecli/pkg/core/result"
)

/*
sanitizeUpdatecliManifestFilePath receives a list of files (directory or file) and returns a list of files that could be accepted by Updatecli.
*/
func sanitizeUpdatecliManifestFilePath(rawFilePaths []string) (sanitizedFilePaths []string) {
for _, r := range rawFilePaths {
err := filepath.Walk(r, func(path string, info os.FileInfo, err error) error {
// sanitizeUpdatecliManifestFilePath receives a list of files (directory or file)
// and returns both a list of files that could be accepted by Updatecli.
// a list of files that can be used as helpers.
func sanitizeUpdatecliManifestFilePath(rawFilePaths []string) (sanitizedFilePaths, sanitizedPartialPaths []string) {
for _, rawFilePath := range rawFilePaths {
rawFileInfo, err := os.Stat(rawFilePath)

// If the manifest if a directory, then we walk trough it to find all manifest files
// and partial files.
if rawFileInfo.IsDir() {
err = filepath.Walk(rawFilePath, func(path string, info os.FileInfo, err error) error {
if err != nil {
logrus.Errorf("\n%s File %s: %s\n", result.FAILURE, path, err)
return fmt.Errorf("unable to walk %q: %s", path, err)
}
if info.Mode().IsRegular() {
baseFile := filepath.Base(path)

if strings.HasPrefix(baseFile, "_") {
sanitizedPartialPaths = append(sanitizedPartialPaths, path)
} else {
sanitizedFilePaths = append(sanitizedFilePaths, path)
}
}
return nil
})
}

// If the manifest is a file, then we check any additional partial files
// in the same directory that start with an underscore.
if rawFileInfo.Mode().IsRegular() {
manifestDirname := filepath.Dir(rawFilePath)
dirEntries, err := os.ReadDir(manifestDirname) // Ensure the directory exists
if err != nil {
logrus.Errorf("\n%s File %s: %s\n", result.FAILURE, path, err)
return fmt.Errorf("unable to walk %q: %s", path, err)
logrus.Errorf("unable to read directory %q: %s", manifestDirname, err)
return nil, nil
}
if info.Mode().IsRegular() {
sanitizedFilePaths = append(sanitizedFilePaths, path)

for _, entry := range dirEntries {
if entry.IsDir() {
continue // Skip directories
}

baseFile := entry.Name()
if strings.HasPrefix(baseFile, "_") {
// If the file starts with an underscore, we consider it a partial file
partialFilePath := filepath.Join(manifestDirname, baseFile)
sanitizedPartialPaths = append(sanitizedPartialPaths, partialFilePath)
}
}
return nil
})

sanitizedFilePaths = append(sanitizedFilePaths, rawFilePath)
}

if err != nil {
logrus.Errorf("err - %s", err)
}
}

// Remove duplicates manifest files
result := []string{}
exist := map[string]bool{}
trimDuplicate := func(input []string) []string {
result := []string{}
exist := map[string]bool{}

for v := range sanitizedFilePaths {
if !exist[sanitizedFilePaths[v]] {
exist[sanitizedFilePaths[v]] = true
result = append(result, sanitizedFilePaths[v])
for v := range input {
if !exist[input[v]] {
exist[input[v]] = true
result = append(result, input[v])
}
}
return result
}

return result
return trimDuplicate(sanitizedFilePaths), trimDuplicate(sanitizedPartialPaths)
}

// PrintTitle print a title
Expand Down

Back | FazBrowse Home | New Git URL