Merge pull request #613 from Morganamilo/parallel

 Parallelize pkgbuild download and use MultiError

download.go

@@ -60,9 +60,11 @@ func gitHasDiff(path string, name string) (bool, error) {
 func gitDownload(url string, path string, name string) (bool, error) {
 	_, err := os.Stat(filepath.Join(path, name, ".git"))
 	if os.IsNotExist(err) {
-		err = show(passToGit(path, "clone", "--no-progress", url, name))
+		cmd := passToGit(path, "clone", "--no-progress", url, name)
+		cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
+		_, stderr, err := capture(cmd)
 		if err != nil {
-			return false, fmt.Errorf("error cloning %s", name)
+			return false, fmt.Errorf("error cloning %s: stderr", name, stderr)
 		}
 
 		return true, nil
@@ -70,23 +72,25 @@ func gitDownload(url string, path string, name string) (bool, error) {
 		return false, fmt.Errorf("error reading %s", filepath.Join(path, name, ".git"))
 	}
 
-	err = show(passToGit(filepath.Join(path, name), "fetch"))
+	cmd := passToGit(filepath.Join(path, name), "fetch")
+	cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
+	_, stderr, err := capture(cmd)
 	if err != nil {
-		return false, fmt.Errorf("error fetching %s", name)
+		return false, fmt.Errorf("error fetching %s: %s", name, stderr)
 	}
 
 	return false, nil
 }
 
 func gitMerge(path string, name string) error {
-	err := show(passToGit(filepath.Join(path, name), "reset", "--hard", "HEAD"))
+	_, stderr, err := capture(passToGit(filepath.Join(path, name), "reset", "--hard", "HEAD"))
 	if err != nil {
-		return fmt.Errorf("error resetting %s", name)
+		return fmt.Errorf("error resetting %s: %s", name, stderr)
 	}
 
-	err = show(passToGit(filepath.Join(path, name), "merge", "--no-edit", "--ff"))
+	_, stderr, err = capture(passToGit(filepath.Join(path, name), "merge", "--no-edit", "--ff"))
 	if err != nil {
-		return fmt.Errorf("error merging %s", name)
+		return fmt.Errorf("error merging %s: %s", name, stderr)
 	}
 
 	return nil

exec.go

@@ -119,6 +119,5 @@ func passToGit(dir string, _args ...string) *exec.Cmd {
 	args = append(args, _args...)
 
 	cmd := exec.Command(config.GitBin, args...)
-	cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr
 	return cmd
 }

install.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 
 	gosrc "github.com/Morganamilo/go-srcinfo"
 	alpm "github.com/jguer/go-alpm"
@@ -770,7 +771,7 @@ func pkgBuildsToSkip(pkgs []*rpc.Pkg, targets stringSet) stringSet {
 			pkgbuild, err := gosrc.ParseFile(dir)
 
 			if err == nil {
-				if alpm.VerCmp(pkgbuild.Version(), pkg.Version) > 0 {
+				if alpm.VerCmp(pkgbuild.Version(), pkg.Version) >= 0 {
 					toSkip.set(pkg.PackageBase)
 				}
 			}
@@ -795,35 +796,57 @@ func mergePkgBuilds(pkgs []*rpc.Pkg) error {
 
 func downloadPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, toSkip stringSet) (stringSet, error) {
 	cloned := make(stringSet)
+	downloaded := 0
+	var wg sync.WaitGroup
+	var mux sync.Mutex
+	var errs MultiError
+
+	download := func(k int, pkg *rpc.Pkg) {
+		defer wg.Done()
 
-	for k, pkg := range pkgs {
 		if toSkip.get(pkg.PackageBase) {
+			mux.Lock()
+			downloaded++
 			str := bold(cyan("::") + " PKGBUILD up to date, Skipping (%d/%d): %s\n")
-			fmt.Printf(str, k+1, len(pkgs), cyan(formatPkgbase(pkg, bases)))
-			continue
+			fmt.Printf(str, downloaded, len(pkgs), cyan(formatPkgbase(pkg, bases)))
+			mux.Unlock()
+			return
 		}
 
-		str := bold(cyan("::") + " Downloading PKGBUILD (%d/%d): %s\n")
-
-		fmt.Printf(str, k+1, len(pkgs), cyan(formatPkgbase(pkg, bases)))
-
 		if shouldUseGit(filepath.Join(config.BuildDir, pkg.PackageBase)) {
 			clone, err := gitDownload(baseURL+"/"+pkg.PackageBase+".git", config.BuildDir, pkg.PackageBase)
 			if err != nil {
-				return nil, err
+				errs.Add(err)
+				return
 			}
 			if clone {
+				mux.Lock()
 				cloned.set(pkg.PackageBase)
+				mux.Unlock()
 			}
 		} else {
 			err := downloadAndUnpack(baseURL+pkg.URLPath, config.BuildDir)
 			if err != nil {
-				return nil, err
+				errs.Add(err)
+				return
 			}
 		}
+
+		mux.Lock()
+		downloaded++
+		str := bold(cyan("::") + " Downloaded PKGBUILD (%d/%d): %s\n")
+		fmt.Printf(str, downloaded, len(pkgs), cyan(formatPkgbase(pkg, bases)))
+		mux.Unlock()
+	}
+
+	for k, pkg := range pkgs {
+		wg.Add(1)
+		go download(k, pkg)
 	}
 
-	return cloned, nil
+	wg.Wait()
+
+	return cloned, errs.Return()
 }
 
 func downloadPkgBuildsSources(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg, incompatible stringSet) (err error) {
@@ -975,10 +998,15 @@ func buildInstallPkgBuilds(dp *depPool, do *depOrder, srcinfos map[string]*gosrc
 			return err
 		}
 
+		var mux sync.Mutex
+		var wg sync.WaitGroup
 		for _, pkg := range do.Bases[pkg.PackageBase] {
-			updateVCSData(pkg.Name, srcinfo.Source)
+			wg.Add(1)
+			go updateVCSData(pkg.Name, srcinfo.Source, &mux, &wg)
 		}
 
+		wg.Wait()
+
 		err = saveVCSInfo()
 		if err != nil {
 			fmt.Println(err)

query.go

@@ -477,16 +477,13 @@ func aurInfo(names []string, warnings *aurWarnings) ([]*rpc.Pkg, error) {
 	seen := make(map[string]int)
 	var mux sync.Mutex
 	var wg sync.WaitGroup
-	var err error
+	var errs MultiError
 
 	makeRequest := func(n, max int) {
 		defer wg.Done()
 		tempInfo, requestErr := rpc.Info(names[n:max])
-		if err != nil {
-			return
-		}
+		errs.Add(requestErr)
 		if requestErr != nil {
-			err = requestErr
 			return
 		}
 		mux.Lock()
@@ -505,7 +502,7 @@ func aurInfo(names []string, warnings *aurWarnings) ([]*rpc.Pkg, error) {
 
 	wg.Wait()
 
-	if err != nil {
+	if err := errs.Return(); err != nil {
 		return info, err
 	}
 

upgrade.go

@@ -3,7 +3,6 @@ package main
 import (
 	"fmt"
 	"sort"
-	"strings"
 	"sync"
 	"unicode"
 
@@ -109,7 +108,7 @@ func getVersionDiff(oldVersion, newVersion string) (left, right string) {
 }
 
 // upList returns lists of packages to upgrade from each source.
-func upList(warnings *aurWarnings) (aurUp upSlice, repoUp upSlice, err error) {
+func upList(warnings *aurWarnings) (upSlice, upSlice, error) {
 	local, remote, _, remoteNames, err := filterPackages()
 	if err != nil {
 		return nil, nil, err
@@ -117,9 +116,10 @@ func upList(warnings *aurWarnings) (aurUp upSlice, repoUp upSlice, err error) {
 
 	var wg sync.WaitGroup
 	var develUp upSlice
+	var repoUp upSlice
+	var aurUp upSlice
 
-	var repoErr error
-	var aurErr error
+	var errs MultiError
 
 	aurdata := make(map[string]*rpc.Pkg)
 
@@ -127,7 +127,8 @@ func upList(warnings *aurWarnings) (aurUp upSlice, repoUp upSlice, err error) {
 		fmt.Println(bold(cyan("::") + bold(" Searching databases for updates...")))
 		wg.Add(1)
 		go func() {
-			repoUp, repoErr = upRepo(local)
+			repoUp, err = upRepo(local)
+			errs.Add(err)
 			wg.Done()
 		}()
 	}
@@ -136,15 +137,17 @@ func upList(warnings *aurWarnings) (aurUp upSlice, repoUp upSlice, err error) {
 		fmt.Println(bold(cyan("::") + bold(" Searching AUR for updates...")))
 
 		var _aurdata []*rpc.Pkg
-		_aurdata, aurErr = aurInfo(remoteNames, warnings)
-		if aurErr == nil {
+		_aurdata, err = aurInfo(remoteNames, warnings)
+		errs.Add(err)
+		if err == nil {
 			for _, pkg := range _aurdata {
 				aurdata[pkg.Name] = pkg
 			}
 
 			wg.Add(1)
 			go func() {
-				aurUp, aurErr = upAUR(remote, aurdata)
+				aurUp, err = upAUR(remote, aurdata)
+				errs.Add(err)
 				wg.Done()
 			}()
 
@@ -163,18 +166,6 @@ func upList(warnings *aurWarnings) (aurUp upSlice, repoUp upSlice, err error) {
 
 	printLocalNewerThanAUR(remote, aurdata)
 
-	errs := make([]string, 0)
-	for _, e := range []error{repoErr, aurErr} {
-		if e != nil {
-			errs = append(errs, e.Error())
-		}
-	}
-
-	if len(errs) > 0 {
-		err = fmt.Errorf("%s", strings.Join(errs, "\n"))
-		return nil, nil, err
-	}
-
 	if develUp != nil {
 		names := make(stringSet)
 		for _, up := range develUp {
@@ -189,7 +180,7 @@ func upList(warnings *aurWarnings) (aurUp upSlice, repoUp upSlice, err error) {
 		aurUp = develUp
 	}
 
-	return aurUp, repoUp, err
+	return aurUp, repoUp, errs.Return()
 }
 
 func upDevel(remote []alpm.Package, aurdata map[string]*rpc.Pkg) (toUpgrade upSlice) {

utils.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"sync"
 	"unicode"
 )
 
@@ -128,3 +129,36 @@ func removeInvalidTargets(targets []string) []string {
 
 	return filteredTargets
 }
+
+type MultiError struct {
+	Errors []error
+	mux    sync.Mutex
+}
+
+func (err *MultiError) Error() string {
+	str := ""
+
+	for _, e := range err.Errors {
+		str += e.Error()
+	}
+
+	return str
+}
+
+func (err *MultiError) Add(e error) {
+	if e == nil {
+		return
+	}
+
+	err.mux.Lock()
+	err.Errors = append(err.Errors, e)
+	err.mux.Unlock()
+}
+
+func (err *MultiError) Return() error {
+	if len(err.Errors) > 0 {
+		return err
+	}
+
+	return nil
+}

vcs.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"sync"
 	"time"
 
 	gosrc "github.com/Morganamilo/go-srcinfo"
@@ -23,6 +24,8 @@ type shaInfo struct {
 
 // createDevelDB forces yay to create a DB of the existing development packages
 func createDevelDB() error {
+	var mux sync.Mutex
+	var wg sync.WaitGroup
 	infoMap := make(map[string]*rpc.Pkg)
 	srcinfosStale := make(map[string]*gosrc.Srcinfo)
 
@@ -46,29 +49,20 @@ func createDevelDB() error {
 	downloadPkgBuilds(info, bases, toSkip)
 	tryParsesrcinfosFile(info, srcinfosStale, bases)
 
-	for _, pkg := range info {
-		pkgbuild, ok := srcinfosStale[pkg.PackageBase]
-		if !ok {
-			continue
-		}
-
-		for _, pkg := range bases[pkg.PackageBase] {
-			updateVCSData(pkg.Name, pkgbuild.Source)
+	for _, pkgbuild := range srcinfosStale {
+		for _, pkg := range pkgbuild.Packages {
+			wg.Add(1)
+			go updateVCSData(pkg.Pkgname, pkgbuild.Source, &mux, &wg)
 		}
 	}
 
+	wg.Wait()
 	fmt.Println(bold(yellow(arrow) + bold(" GenDB finished. No packages were installed")))
-
 	return err
 }
 
 // parseSource returns the git url, default branch and protocols it supports
 func parseSource(source string) (url string, branch string, protocols []string) {
-	if !(strings.Contains(source, "git://") ||
-		strings.Contains(source, ".git") ||
-		strings.Contains(source, "git+https://")) {
-		return "", "", nil
-	}
 	split := strings.Split(source, "::")
 	source = split[len(split)-1]
 	split = strings.SplitN(source, "://", 2)
@@ -76,8 +70,20 @@ func parseSource(source string) (url string, branch string, protocols []string)
 	if len(split) != 2 {
 		return "", "", nil
 	}
-
 	protocols = strings.Split(split[0], "+")
+
+	git := false
+	for _, protocol := range protocols {
+		if protocol == "git" {
+			git = true
+			break
+		}
+	}
+
+	if !git {
+		return "", "", nil
+	}
+
 	split = strings.SplitN(split[1], "#", 2)
 	if len(split) == 2 {
 		secondSplit := strings.SplitN(split[1], "=", 2)
@@ -102,24 +108,29 @@ func parseSource(source string) (url string, branch string, protocols []string)
 	return
 }
 
-func updateVCSData(pkgName string, sources []gosrc.ArchString) {
+func updateVCSData(pkgName string, sources []gosrc.ArchString, mux *sync.Mutex, wg *sync.WaitGroup) {
+	defer wg.Done()
+
 	if savedInfo == nil {
+		mux.Lock()
 		savedInfo = make(vcsInfo)
+		mux.Unlock()
 	}
 
 	info := make(shaInfos)
-
-	for _, source := range sources {
+	checkSource := func(source gosrc.ArchString) {
+		defer wg.Done()
 		url, branch, protocols := parseSource(source.Value)
 		if url == "" || branch == "" {
-			continue
+			return
 		}
 
 		commit := getCommit(url, branch, protocols)
 		if commit == "" {
-			continue
+			return
 		}
 
+		mux.Lock()
 		info[url] = shaInfo{
 			protocols,
 			branch,
@@ -127,9 +138,14 @@ func updateVCSData(pkgName string, sources []gosrc.ArchString) {
 		}
 
 		savedInfo[pkgName] = info
-
 		fmt.Println(bold(yellow(arrow)) + " Found git repo: " + cyan(url))
 		saveVCSInfo()
+		mux.Unlock()
+	}
+
+	for _, source := range sources {
+		wg.Add(1)
+		go checkSource(source)
 	}
 }
 
@@ -152,6 +168,7 @@ func getCommit(url string, branch string, protocols []string) string {
 		//Introduce a time out so this can not hang
 		timer := time.AfterFunc(5*time.Second, func() {
 			cmd.Process.Kill()
+			fmt.Println(bold(yellow(arrow)), "Timeout:", cyan(url))
 		})
 
 		err = cmd.Wait()