Merge pull request #641 from Morganamilo/parallel-G

 Use goroutines for fetching packages via -G

doc/yay.8

@@ -31,7 +31,8 @@ Perform yay specific print operations.
 
 .TP
 .B \-G, \-\-getpkgbuild
-Downloads PKGBUILD from ABS or AUR.
+Downloads PKGBUILD from ABS or AUR. ABS pkgbuilds are always downloaded using
+tarballs and taken from trunk. The ABS can only be used for Arch Linux repositories
 
 .RE
 If no arguments are provided 'yay \-Syu' will be performed.
@@ -276,7 +277,7 @@ Show a detailed list of updates in a similar format to VerbosePkgLists.
 Upgrades can also be skipped using numbers, number ranges or repo names.
 Adidionally ^ can be used to invert the selection.
 
-\fBWarning\fR: It is not recommended to skip updates from the repositores as
+\fBWarning\fR: It is not recommended to skip updates from the repositories as
 this can lead to partial upgrades. This feature is intended to easily skip AUR
 updates on the fly that may be broken or have a long compile time. Ultimately
 it is up to the user what upgrades they skip.

download.go

@@ -8,6 +8,9 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"
+
+	alpm "github.com/jguer/go-alpm"
 )
 
 // Decide what download method to use:
@@ -103,10 +106,10 @@ func gitDiff(path string, name string) error {
 }
 
 // DownloadAndUnpack downloads url tgz and extracts to path.
-func downloadAndUnpack(url string, path string) (err error) {
-	err = os.MkdirAll(path, 0755)
+func downloadAndUnpack(url string, path string) error {
+	err := os.MkdirAll(path, 0755)
 	if err != nil {
-		return
+		return err
 	}
 
 	fileName := filepath.Base(url)
@@ -116,15 +119,15 @@ func downloadAndUnpack(url string, path string) (err error) {
 
 	err = downloadFile(tarLocation, url)
 	if err != nil {
-		return
+		return err
 	}
 
-	err = exec.Command(config.TarBin, "-xf", tarLocation, "-C", path).Run()
+	_, stderr, err := capture(exec.Command(config.TarBin, "-xf", tarLocation, "-C", path))
 	if err != nil {
-		return
+		return fmt.Errorf("%s", stderr)
 	}
 
-	return
+	return nil
 }
 
 func getPkgbuilds(pkgs []string) error {
@@ -135,9 +138,18 @@ func getPkgbuilds(pkgs []string) error {
 	}
 
 	pkgs = removeInvalidTargets(pkgs)
-
 	aur, repo, err := packageSlices(pkgs)
 
+	for n := range aur {
+		_, pkg := splitDbFromName(aur[n])
+		aur[n] = pkg
+	}
+
+	info, err := aurInfoPrint(aur)
+	if err != nil {
+		return err
+	}
+
 	if len(repo) > 0 {
 		missing, err = getPkgbuildsfromABS(repo, wd)
 		if err != nil {
@@ -146,11 +158,13 @@ func getPkgbuilds(pkgs []string) error {
 	}
 
 	if len(aur) > 0 {
-		_missing, err := getPkgbuildsfromAUR(aur, wd)
-		if err != nil {
+		bases := getBases(info)
+		toSkip := pkgbuildsToSkip(bases, nil)
+		if _, err = downloadPkgbuilds(bases, toSkip, wd); err != nil {
 			return err
 		}
-		missing = missing || _missing
+
+		missing = missing || len(aur) != len(info)
 	}
 
 	if missing {
@@ -161,107 +175,94 @@ func getPkgbuilds(pkgs []string) error {
 }
 
 // GetPkgbuild downloads pkgbuild from the ABS.
-func getPkgbuildsfromABS(pkgs []string, path string) (missing bool, err error) {
+func getPkgbuildsfromABS(pkgs []string, path string) (bool, error) {
+	var wg sync.WaitGroup
+	var mux sync.Mutex
+	var errs MultiError
+	names := make(map[string]string)
+	missing := make([]string, 0)
+	downloaded := 0
+
 	dbList, err := alpmHandle.SyncDbs()
 	if err != nil {
-		return
+		return false, err
 	}
 
-nextPkg:
 	for _, pkgN := range pkgs {
+		var pkg *alpm.Package
+		var err error
+		var url string
 		pkgDb, name := splitDbFromName(pkgN)
 
-		for _, db := range dbList.Slice() {
-			if pkgDb != "" && db.Name() != pkgDb {
-				continue
+		if pkgDb != "" {
+			if db, err := alpmHandle.SyncDbByName(pkgDb); err == nil {
+				pkg, err = db.PkgByName(name)
 			}
-
-			pkg, err := db.PkgByName(name)
-			if err == nil {
-				var url string
-				name := pkg.Base()
-				if name == "" {
-					name = pkg.Name()
-				}
-
-				if _, err := os.Stat(filepath.Join(path, name)); err == nil {
-					fmt.Println(bold(red(arrow)), bold(cyan(name)), "directory already exists")
-					continue nextPkg
-				}
-
-				switch db.Name() {
-				case "core", "extra":
-					url = "https://git.archlinux.org/svntogit/packages.git/snapshot/packages/" + name + ".tar.gz"
-				case "community", "multilib":
-					url = "https://git.archlinux.org/svntogit/community.git/snapshot/packages/" + name + ".tar.gz"
-				default:
-					fmt.Println(pkgN, "not in standard repositories")
-					continue nextPkg
-				}
-
-				errD := downloadAndUnpack(url, cacheHome)
-				if errD != nil {
-					fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name())), bold(red(errD.Error())))
-				}
-
-				errD = exec.Command("mv", filepath.Join(cacheHome, "packages", name, "trunk"), filepath.Join(path, name)).Run()
-				if errD != nil {
-					fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name())), bold(red(errD.Error())))
-				} else {
-					fmt.Println(bold(yellow(arrow)), "Downloaded", cyan(pkg.Name()), "from ABS")
+		} else {
+			dbList.ForEach(func(db alpm.Db) error {
+				if pkg, err = db.PkgByName(name); err == nil {
+					return fmt.Errorf("")
 				}
+				return nil
+			})
+		}
 
-				continue nextPkg
-			}
+		if pkg == nil {
+			missing = append(missing, name)
+			continue
 		}
 
-		fmt.Println(pkgN, "could not find package in database")
-		missing = true
-	}
+		name = pkg.Base()
+		if name == "" {
+			name = pkg.Name()
+		}
 
-	if _, err := os.Stat(filepath.Join(cacheHome, "packages")); err == nil {
-		os.RemoveAll(filepath.Join(cacheHome, "packages"))
-	}
+		switch pkg.DB().Name() {
+		case "core", "extra", "testing":
+			url = "https://git.archlinux.org/svntogit/packages.git/snapshot/packages/" + name + ".tar.gz"
+		case "community", "multilib", "community-testing", "multilib-testing":
+			url = "https://git.archlinux.org/svntogit/community.git/snapshot/packages/" + name + ".tar.gz"
+		default:
+			missing = append(missing, name)
+			continue
+		}
 
-	return
-}
+		if err = os.RemoveAll(filepath.Join(path, name)); err != nil {
+			fmt.Println(bold(red(smallArrow)), err)
+			continue
+		}
 
-// GetPkgbuild downloads pkgbuild from the AUR.
-func getPkgbuildsfromAUR(pkgs []string, dir string) (bool, error) {
-	missing := false
-	strippedPkgs := make([]string, 0)
-	for _, pkg := range pkgs {
-		_, name := splitDbFromName(pkg)
-		strippedPkgs = append(strippedPkgs, name)
+		names[name] = url
 	}
 
-	aq, err := aurInfoPrint(strippedPkgs)
-	if err != nil {
-		return missing, err
+	if len(missing) != 0 {
+		fmt.Println(yellow(bold(smallArrow)), "Missing ABS packages: ", cyan(strings.Join(missing, "  ")))
 	}
 
-	for _, pkg := range aq {
-		if _, err := os.Stat(filepath.Join(dir, pkg.PackageBase)); err == nil {
-			fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name)), "directory already exists")
-			continue
-		}
-
-		if shouldUseGit(filepath.Join(dir, pkg.PackageBase)) {
-			_, err = gitDownload(baseURL+"/"+pkg.PackageBase+".git", dir, pkg.PackageBase)
-		} else {
-			err = downloadAndUnpack(baseURL+aq[0].URLPath, dir)
+	download := func(pkg string, url string) {
+		defer wg.Done()
+		if err := downloadAndUnpack(url, cacheHome); err != nil {
+			errs.Add(fmt.Errorf("%s Failed to get pkgbuild: %s: %s", bold(red(arrow)), bold(cyan(pkg)), bold(red(err.Error()))))
+			return
 		}
 
+		_, stderr, err := capture(exec.Command("mv", filepath.Join(cacheHome, "packages", pkg, "trunk"), filepath.Join(path, pkg)))
+		mux.Lock()
+		downloaded++
 		if err != nil {
-			fmt.Println(err)
+			errs.Add(fmt.Errorf("%s Failed to move %s: %s", bold(red(arrow)), bold(cyan(pkg)), bold(red(string(stderr)))))
 		} else {
-			fmt.Println(bold(yellow(arrow)), "Downloaded", cyan(pkg.PackageBase), "from AUR")
+			fmt.Printf(bold(cyan("::"))+" Downloaded PKGBUILD from ABS (%d/%d): %s\n", downloaded, len(names), cyan(pkg))
 		}
+		mux.Unlock()
 	}
 
-	if len(aq) != len(pkgs) {
-		missing = true
+	for name, url := range names {
+		wg.Add(1)
+		go download(name, url)
 	}
 
-	return missing, err
+	wg.Wait()
+	errs.Add(os.RemoveAll(filepath.Join(cacheHome, "packages")))
+	return len(missing) != 0, errs.Return()
 }

exec.go

@@ -25,8 +25,8 @@ func capture(cmd *exec.Cmd) (string, string, error) {
 	cmd.Stdout = &outbuf
 	cmd.Stderr = &errbuf
 	err := cmd.Run()
-	stdout := outbuf.String()
-	stderr := errbuf.String()
+	stdout := strings.TrimSpace(outbuf.String())
+	stderr := strings.TrimSpace(errbuf.String())
 
 	return stdout, stderr, err
 }

install.go

@@ -185,7 +185,7 @@ func install(parser *arguments) error {
 	}
 
 	toSkip := pkgbuildsToSkip(do.Aur, targets)
-	cloned, err := downloadPkgbuilds(do.Aur, toSkip)
+	cloned, err := downloadPkgbuilds(do.Aur, toSkip, config.BuildDir)
 	if err != nil {
 		return err
 	}
@@ -809,7 +809,7 @@ func mergePkgbuilds(bases []Base) error {
 	return nil
 }
 
-func downloadPkgbuilds(bases []Base, toSkip stringSet) (stringSet, error) {
+func downloadPkgbuilds(bases []Base, toSkip stringSet, buildDir string) (stringSet, error) {
 	cloned := make(stringSet)
 	downloaded := 0
 	var wg sync.WaitGroup
@@ -830,7 +830,7 @@ func downloadPkgbuilds(bases []Base, toSkip stringSet) (stringSet, error) {
 		}
 
 		if shouldUseGit(filepath.Join(config.BuildDir, pkg)) {
-			clone, err := gitDownload(baseURL+"/"+pkg+".git", config.BuildDir, pkg)
+			clone, err := gitDownload(baseURL+"/"+pkg+".git", buildDir, pkg)
 			if err != nil {
 				errs.Add(err)
 				return
@@ -841,7 +841,7 @@ func downloadPkgbuilds(bases []Base, toSkip stringSet) (stringSet, error) {
 				mux.Unlock()
 			}
 		} else {
-			err := downloadAndUnpack(baseURL+base.URLPath(), config.BuildDir)
+			err := downloadAndUnpack(baseURL+base.URLPath(), buildDir)
 			if err != nil {
 				errs.Add(err)
 				return

utils.go

@@ -139,10 +139,10 @@ func (err *MultiError) Error() string {
 	str := ""
 
 	for _, e := range err.Errors {
-		str += e.Error()
+		str += e.Error() + "\n"
 	}
 
-	return str
+	return str[:len(str)-1]
 }
 
 func (err *MultiError) Add(e error) {

vcs.go

@@ -38,7 +38,7 @@ func createDevelDB() error {
 
 	bases := getBases(info)
 	toSkip := pkgbuildsToSkip(bases, sliceToStringSet(remoteNames))
-	downloadPkgbuilds(bases, toSkip)
+	downloadPkgbuilds(bases, toSkip, config.BuildDir)
 	srcinfos, _ := parseSrcinfoFiles(bases, false)
 
 	for _, pkgbuild := range srcinfos {