DataHearth
/
config-mapper
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Activity

feat(index): add indexing system

This commit is contained in:
DataHearth 2022-04-07 20:08:44 +02:00
parent a85165e8fe
commit 584fe09fe2
No known key found for this signature in database
GPG Key ID: E88FD356ACC5F3C4
10 changed files with 371 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,3 +1,4 @@
.config-mapper.yml
demo
build
build
.DS_Store

91
cmd/cli.go
View File

@ -1,14 +1,13 @@
package cmd
import (
"fmt"
"log"
"os"
"strconv"
"time"
mapper "github.com/datahearth/config-mapper/internal"
"github.com/pterm/pterm"
"github.com/fatih/color"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
@ -34,14 +33,14 @@ var initCmd = &cobra.Command{
var config mapper.Configuration
if err := viper.Unmarshal(&config); err != nil {
errLogger.Printf(pterm.Red(fmt.Sprintf("failed to decode configuration: %v\n", err)))
mapper.PrintError("failed to decode configuration: %v\n", err)
os.Exit(1)
}
logger.Println("initializing config-mapper folder from configuration...")
if _, err := mapper.NewRepository(config.Storage.Git, config.Storage.Path); err != nil {
errLogger.Printf(pterm.Red(fmt.Sprintf("failed to initialize folder: %v\n", err)))
mapper.PrintError("failed to initialize folder: %v\n", err)
os.Exit(1)
}
@ -54,30 +53,38 @@ var loadCmd = &cobra.Command{
Long: `Load your files, folders and package managers deps configurations onto your new
onto your new system based on your configuration file`,
Run: func(cmd *cobra.Command, args []string) {
var config mapper.Configuration
if err := viper.Unmarshal(&config); err != nil {
errLogger.Printf(pterm.Red(fmt.Sprintf("failed to decode configuration: %v\n", err)))
var c mapper.Configuration
if err := viper.Unmarshal(&c); err != nil {
mapper.PrintError("failed to decode configuration: %v\n", err)
os.Exit(1)
}
el := mapper.NewElement([]mapper.ItemLocation{}, config.Storage.Path)
i, err := mapper.NewIndexer(c.Storage.Path)
if err != nil {
mapper.PrintError("failed to open the indexer: %v\n", err)
os.Exit(1)
}
r, err := mapper.NewRepository(c.Storage.Git, c.Storage.Path)
if err != nil {
mapper.PrintError("failed to open repository at %s: %v\n", c.Storage.Path, err)
os.Exit(1)
}
el := mapper.NewItemsActions(nil, c.Storage.Path, r, i)
if !viper.GetBool("load-disable-files") {
el.AddItems(config.Files)
el.AddItems(c.Files)
}
if !viper.GetBool("load-disable-folders") {
el.AddItems(config.Folders)
el.AddItems(c.Folders)
}
if err := el.Action("load"); err != nil {
errLogger.Printf(pterm.Red(err))
os.Exit(1)
}
el.Action("load")
if !viper.GetBool("load-disable-pkgs") {
if err := mapper.LoadPkgs(config.PackageManagers); err != nil {
errLogger.Printf(pterm.Red(err))
if err := mapper.LoadPkgs(c.PackageManagers); err != nil {
mapper.PrintError(err.Error())
os.Exit(1)
}
}
@ -89,50 +96,56 @@ var saveCmd = &cobra.Command{
Long: `Save your files, folders and package managers deps configurations onto your
saved location based on your configuration file`,
Run: func(cmd *cobra.Command, args []string) {
var config mapper.Configuration
if err := viper.Unmarshal(&config); err != nil {
errLogger.Printf(pterm.Red(fmt.Sprintf("failed to decode configuration: %v\n", err)))
var c mapper.Configuration
if err := viper.Unmarshal(&c); err != nil {
mapper.PrintError("failed to decode configuration: %v\n", err)
os.Exit(1)
}
repo, err := mapper.NewRepository(config.Storage.Git, config.Storage.Path)
indexer, err := mapper.NewIndexer(c.Storage.Path)
if err != nil {
errLogger.Printf(pterm.Red(fmt.Sprintf("failed to open repository at %s: %v\n", config.Storage.Path, err)))
mapper.PrintError("failed to open the indexer: %v\n", err)
os.Exit(1)
}
el := mapper.NewElement([]mapper.ItemLocation{}, config.Storage.Path)
r, err := mapper.NewRepository(c.Storage.Git, c.Storage.Path)
if err != nil {
mapper.PrintError("failed to open repository at %s: %v\n", c.Storage.Path, err)
os.Exit(1)
}
el := mapper.NewItemsActions(nil, c.Storage.Path, r, indexer)
if !viper.GetBool("save-disable-files") {
el.AddItems(config.Files)
el.AddItems(c.Files)
}
if !viper.GetBool("save-disable-folders") {
el.AddItems(config.Folders)
el.AddItems(c.Folders)
}
if err := el.Action("save"); err != nil {
errLogger.Printf(pterm.Red(err))
os.Exit(1)
}
el.Action("save")
if !viper.GetBool("save-disable-pkgs") {
if err := mapper.SavePkgs(config); err != nil {
errLogger.Printf(pterm.Red(err))
if err := mapper.SavePkgs(c); err != nil {
mapper.PrintError(err.Error())
os.Exit(1)
}
}
if err := el.CleanUp(indexer.RemovedLines()); err != nil {
mapper.PrintError("failed to clean repository: %v\n", err)
os.Exit(1)
}
if viper.GetBool("push") {
pterm.DefaultSection.Println("Pushing items")
color.Blue("# Pushing items")
s, _ := pterm.DefaultSpinner.WithShowTimer(true).WithRemoveWhenDone(false).Start("Pushing changes to remote repository")
if err := repo.PushChanges(viper.GetString("message")); err != nil {
errLogger.Printf(pterm.Red(fmt.Sprintf("failed to push changes to repository: %v\n", err)))
if err := r.PushChanges(viper.GetString("message"), indexer.Lines(), indexer.RemovedLines()); err != nil {
mapper.PrintError("failed to push changes to repository: %v\n", err)
os.Exit(1)
}
s.Stop()
s.Success("Changes pushed")
color.Green("Items pushed")
}
},
}
@ -158,10 +171,12 @@ func init() {
saveCmd.PersistentFlags().Bool("disable-pkgs", false, "package managers will be ignored")
saveCmd.Flags().BoolP("push", "p", false, "new configurations will be committed and pushed")
saveCmd.Flags().StringP("message", "m", strconv.FormatInt(time.Now().Unix(), 10), "combined with --push to set a commit message")
saveCmd.Flags().Bool("disable-index", false, "configuration index will not be updated")
viper.BindPFlag("save-disable-files", saveCmd.PersistentFlags().Lookup("disable-files"))
viper.BindPFlag("save-disable-folders", saveCmd.PersistentFlags().Lookup("disable-folders"))
viper.BindPFlag("save-disable-pkgs", saveCmd.PersistentFlags().Lookup("disable-pkgs"))
viper.BindPFlag("push", saveCmd.Flags().Lookup("push"))
viper.BindPFlag("disable-index-update", saveCmd.Flags().Lookup("disable-index"))
viper.BindPFlag("message", saveCmd.Flags().Lookup("message"))
}

6
go.mod
View File

@ -10,12 +10,18 @@ require (
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)
require (
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
)
require (
github.com/Microsoft/go-winio v0.4.16 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect
github.com/atomicgo/cursor v0.0.1 // indirect
github.com/emirpasic/gods v1.12.0 // indirect
github.com/fatih/color v1.13.0
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-git/gcfg v1.5.0 // indirect
github.com/go-git/go-billy/v5 v5.3.1 // indirect

3
go.sum
View File

@ -126,6 +126,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
@ -303,12 +304,14 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=

5
internal/configuration.go
View File

@ -6,7 +6,6 @@ import (
"path"
"strings"
"github.com/pterm/pterm"
"github.com/spf13/viper"
)
@ -82,9 +81,9 @@ func InitConfig() {
if err := viper.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); ok {
errLogger.Println(pterm.Red(err))
PrintError(err.Error())
} else {
errLogger.Printf(pterm.Red(fmt.Sprintf("failed to read config: %v\n", err)))
PrintError("failed to read config: %v\n", err)
}
os.Exit(1)

136
internal/element.go
View File

@ -1,136 +0,0 @@
package mapper
import (
"errors"
"fmt"
"os"
"path"
"strings"
"github.com/pterm/pterm"
)
var (
ErrCopy = errors.New("failed to copy some files")
ErrUnsupportedOS = errors.New("unsupported OS. Please, contact the maintainer")
)
type Items struct {
locations []ItemLocation
storage string
progessBar *pterm.ProgressbarPrinter
runErr bool
}
type ItemsActions interface {
Action(action string) error
AddItems(items []ItemLocation)
copy(src, dst string) error
}
func NewElement(l []ItemLocation, storage string) ItemsActions {
return &Items{
l,
storage,
pterm.DefaultProgressbar.WithTotal(len(l)),
false,
}
}
func (e *Items) Action(a string) error {
pterm.DefaultSection.Println(fmt.Sprintf("%s items", strings.Title(a)))
e.progessBar.Start()
for _, f := range e.locations {
var src, dst string
storagePath, systemPath, err := configPaths(f, e.storage)
if err != nil {
pterm.Error.Println(fmt.Sprintf("failed to resolve item paths \"%v\": %v", f, err))
e.runErr = true
continue
}
if a == "save" {
src = systemPath
dst = storagePath
} else {
src = storagePath
dst = systemPath
}
if err := os.MkdirAll(path.Dir(dst), 0755); err != nil {
pterm.Error.Printfln(fmt.Sprintf("failed to create directory architecture for destination path \"%s\": %v", path.Dir(dst), err))
e.runErr = true
continue
}
s, err := os.Stat(src)
if err != nil {
pterm.Error.Println(fmt.Sprintf("failed to check if source path is a folder \"%s\": %v", src, err))
e.runErr = true
continue
}
e.progessBar.UpdateTitle(fmt.Sprintf("copying %s", src))
if s.IsDir() {
if err := os.Mkdir(dst, 0755); err != nil {
if !os.IsExist(err) {
pterm.Error.Println(fmt.Sprintf("failed to create destination folder \"%s\": %v", dst, err))
e.runErr = true
continue
}
}
if err := copyFolder(src, dst); err != nil {
pterm.Error.Println(fmt.Sprintf("failed to %s folder from \"%s\" to \"%s\": %v", a, src, dst, err))
e.runErr = true
continue
}
} else {
if err := copyFile(src, dst); err != nil {
pterm.Error.Println(fmt.Sprintf("failed to %s file from \"%s\" to \"%s\": %v", a, src, dst, err))
e.runErr = true
continue
}
}
pterm.Success.Println(fmt.Sprintf("%s copied", src))
e.progessBar.Increment()
}
e.progessBar.Stop()
if e.runErr {
e.runErr = false
return ErrCopy
}
return nil
}
func (e *Items) AddItems(items []ItemLocation) {
e.locations = append(e.locations, items...)
}
func (e *Items) copy(src, dst string) error {
s, err := os.Stat(src)
if err != nil {
return err
}
e.progessBar.UpdateTitle(fmt.Sprintf("copying %s", src))
if s.IsDir() {
if err := copyFolder(src, dst); err != nil {
return err
}
} else {
if err := copyFile(src, dst); err != nil {
return err
}
}
pterm.Success.Println(fmt.Sprintf("\"%s\" copied", src))
e.progessBar.Increment()
return nil
}

70
internal/git.go
View File

@ -3,7 +3,6 @@ package mapper
import (
"errors"
"os"
"os/exec"
"time"
"github.com/go-git/go-git/v5"
@ -19,7 +18,9 @@ var (
)
type RepositoryActions interface {
PushChanges(msg string) error
PushChanges(msg string, newLines, removedLines []string) error
GetWorktree() (*git.Worktree, error)
GetAuthor() *object.Signature
openRepository() error
}
@ -114,38 +115,61 @@ func (r *Repository) openRepository() error {
return err
}
w, err := repo.Worktree()
if err != nil {
return err
}
err = w.Pull(&git.PullOptions{
Auth: r.auth,
})
if err != nil && err != git.NoErrAlreadyUpToDate {
return err
}
r.repository = repo
return nil
}
func (r *Repository) PushChanges(msg string) error {
func (r *Repository) PushChanges(msg string, newLines, removedLines []string) error {
w, err := r.repository.Worktree()
if err != nil {
return err
}
// TODO: debug why deleted files/folders aren't added to index
// if err := w.AddWithOptions(&git.AddOptions{
// All: true,
for _, l := range newLines {
if err := w.AddWithOptions(&git.AddOptions{
Path: l,
}); err != nil {
return err
}
}
for _, l := range removedLines {
if err := w.AddWithOptions(&git.AddOptions{
Path: l,
}); err != nil {
return err
}
}
return nil
// if _, err := w.Commit(msg, &git.CommitOptions{
// Author: r.GetAuthor(),
// }); err != nil {
// return err
// }
cmd := exec.Command("git", "add", ".")
cmd.Dir = r.repoPath
if err := cmd.Run(); err != nil {
return errors.New("failed to add files to git index: " + err.Error())
}
if _, err := w.Commit(msg, &git.CommitOptions{
Author: &object.Signature{
Name: r.author.name,
Email: r.author.email,
When: time.Now(),
},
}); err != nil {
return err
}
return r.repository.Push(&git.PushOptions{})
// return r.repository.Push(&git.PushOptions{})
}
func (r *Repository) GetWorktree() (*git.Worktree, error) {
return r.repository.Worktree()
}
func (r *Repository) GetAuthor() *object.Signature {
return &object.Signature{
Name: r.author.name,
Email: r.author.email,
When: time.Now(),
}
}

105
internal/index.go Normal file
View File

@ -0,0 +1,105 @@
package mapper
import (
"fmt"
"io/fs"
"os"
"strings"
)
type Index struct {
lines []string
path string
perms fs.FileMode
repoPath string
removedLines []string
}
type Indexer interface {
Write(newLines []string) error
filter(configLines []string) map[string]bool
RemovedLines() []string
Lines() []string
}
func NewIndexer(repoPath string) (Indexer, error) {
perms := fs.FileMode(0755)
indexPath, err := absolutePath(fmt.Sprintf("%s/.index", repoPath))
if err != nil {
return nil, err
}
var l []string
s, err := os.Stat(indexPath)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
l = []string{}
} else {
perms = s.Mode()
b, err := os.ReadFile(indexPath)
if err != nil {
return nil, err
}
l = strings.Split(string(b), "\n")
}
return &Index{
lines: l,
path: indexPath,
perms: perms,
repoPath: repoPath,
removedLines: []string{},
}, nil
}
func (i *Index) RemovedLines() []string {
return i.removedLines
}
func (i *Index) Lines() []string {
return i.lines
}
// filter removes lines that are no more used in configuration from the index
func (i *Index) filter(newLines []string) map[string]bool {
removedLines := []string{}
foundLines := map[string]bool{}
for _, nl := range newLines {
foundLines[nl] = true
}
for _, ml := range i.lines {
if _, ok := foundLines[ml]; !ok {
removedLines = append(removedLines, ml)
}
}
i.removedLines = removedLines
return foundLines
}
// Write add lines stored in memory the .index file
func (i *Index) Write(newLines []string) error {
lines := i.filter(newLines)
var data []byte
index := 0
linesNumber := len(lines)
for path := range lines {
if index+1 == linesNumber {
data = append(data, []byte(fmt.Sprint(path))...)
} else {
data = append(data, []byte(fmt.Sprintln(path))...)
}
index += 1
}
os.WriteFile(i.path, data, i.perms)
return nil
}

137
internal/items.go Normal file
View File

@ -0,0 +1,137 @@
package mapper
import (
"fmt"
"io/fs"
"os"
"path"
"strings"
"github.com/fatih/color"
"github.com/spf13/viper"
)
type Items struct {
locations []ItemLocation
storage string
repository RepositoryActions
indexer Indexer
}
type ItemsActions interface {
Action(action string)
AddItems(items []ItemLocation)
CleanUp(removedLines []string) error
}
func NewItemsActions(items []ItemLocation, storage string, repository RepositoryActions, indexer Indexer) ItemsActions {
if items == nil {
items = []ItemLocation{}
}
return &Items{
locations: items,
storage: storage,
repository: repository,
indexer: indexer,
}
}
func (e *Items) Action(a string) {
color.Blue("# %s", a)
newLines := []string{}
for i, l := range e.locations {
var src, dst string
storagePath, systemPath, err := configPaths(l, e.storage)
if err != nil {
PrintError("[%d] failed to resolve item paths \"%v\": %v", i, l, err)
continue
}
if a == "save" {
src = systemPath
dst = storagePath
} else {
src = storagePath
dst = systemPath
}
if err := os.MkdirAll(path.Dir(dst), 0755); err != nil {
PrintError("[%d] failed to create directory architecture for destination path \"%s\": %v", i, path.Dir(dst), err)
continue
}
s, err := os.Stat(src)
if err != nil {
PrintError("[%d] failed to check if source path is a folder \"%s\": %v", i, src, err)
continue
}
if s.IsDir() {
dstPerms := fs.FileMode(0755)
s, err := os.Stat(dst)
if err != nil {
if !os.IsNotExist(err) {
PrintError("[%d] failed to check if destination folder \"%s\" exists: %v", i, dst, err)
continue
}
} else {
dstPerms = s.Mode()
}
if err := os.Mkdir(dst, dstPerms); err != nil {
if !os.IsExist(err) {
PrintError("[%d] failed to create destination folder \"%s\": %v", i, dst, err)
continue
}
}
if err := copyFolder(src, dst); err != nil {
PrintError("[%d] failed to %s folder from \"%s\" to \"%s\": %v", i, a, src, dst, err)
continue
}
} else {
if err := copyFile(src, dst); err != nil {
PrintError("[%d] failed to %s file from \"%s\" to \"%s\": %v", i, a, src, dst, err)
continue
}
}
if a == "save" {
p, err := absolutePath(e.storage)
if err != nil {
PrintError("[%d] failed resolve absolute path from configuration storage: %v", i, err)
continue
}
newLines = append(newLines, strings.ReplaceAll(dst, p+"/", ""))
}
color.Green("[%d] %s copied", i, src)
}
if a == "save" && !viper.GetBool("disable-index-update") {
if err := e.indexer.Write(newLines); err != nil {
PrintError(err.Error())
os.Exit(1)
}
}
}
func (e *Items) AddItems(items []ItemLocation) {
e.locations = append(e.locations, items...)
}
func (e *Items) CleanUp(removedLines []string) error {
for _, l := range removedLines {
path, err := absolutePath(fmt.Sprintf("%s/%s", e.storage, l))
if err != nil {
return err
}
if err := os.Remove(path); err != nil {
return fmt.Errorf("failed to remove item %s: %v", l, err)
}
}
return nil
}

18
internal/tools.go
View File

@ -1,12 +1,15 @@
package mapper
import (
"errors"
"fmt"
"io"
"os"
"path"
"runtime"
"strings"
"github.com/fatih/color"
)
func absolutePath(p string) (string, error) {
@ -99,7 +102,7 @@ func configPaths(f ItemLocation, location string) (string, string, error) {
return "", "", err
}
default:
return "", "", ErrUnsupportedOS
return "", "", errors.New("unsupported OS. Please, contact the maintainer")
}
return src, dst, nil
@ -117,12 +120,19 @@ func copyFolder(src, dst string) error {
dstItem := fmt.Sprintf("%s/%s", dst, itemName)
if i.IsDir() {
if err := os.Mkdir(dstItem, i.Type().Perm()); err != nil {
info, err := i.Info()
if err != nil {
return err
}
if err := os.MkdirAll(dstItem, info.Mode()); err != nil {
return err
}
if err := copyFolder(srcItem, dstItem); err != nil {
return err
}
continue
}
if err := copyFile(srcItem, dstItem); err != nil {
@ -132,3 +142,7 @@ func copyFolder(src, dst string) error {
return nil
}
func PrintError(err string, values ...interface{}) {
color.Error.Write([]byte(fmt.Sprintf(err, values...)))
}