Worker builds binary in case the binary doesn't match the worker arch

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/emirpasic/gods v1.9.0 // indirect
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
 	github.com/gaia-pipeline/flag v1.7.4-pre
+	github.com/gaia-pipeline/gosdk v0.0.0-20180909192508-cc9f89055777
 	github.com/gaia-pipeline/protobuf v0.0.0-20180812091451-7be8a901b55a
 	github.com/gliderlabs/ssh v0.1.1 // indirect
 	github.com/golang/protobuf v1.3.1
@@ -39,6 +40,7 @@ require (
 	github.com/stretchr/testify v1.3.0 // indirect
 	github.com/xanzy/ssh-agent v0.2.0 // indirect
 	golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67
+	golang.org/x/net v0.0.0-20180826012351-8a410e7b638d
 	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
 	golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 // indirect
 	google.golang.org/grpc v1.18.0

go.sum

@@ -30,6 +30,8 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjr
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/gaia-pipeline/flag v1.7.4-pre h1:/TAmHVYVQGE4Mw9xl0Qs0D5UruVDMF95thexyEFbTAY=
 github.com/gaia-pipeline/flag v1.7.4-pre/go.mod h1:rLpsWzqOEPa2K0Yl4aC34nmblLpIYjGqjP/srZbYvEk=
+github.com/gaia-pipeline/gosdk v0.0.0-20180909192508-cc9f89055777 h1:Gn3nmETr4IE44pIFLeTDNNBBRhNXXh53PqFV23CsVok=
+github.com/gaia-pipeline/gosdk v0.0.0-20180909192508-cc9f89055777/go.mod h1:e3TkvPdcdSdHZTgwiS89fs8lJrveYHsGLlz/Q0oXlN8=
 github.com/gaia-pipeline/protobuf v0.0.0-20180812091451-7be8a901b55a h1:/5XAmdAyGl4yL9BugdPdBLaXquif1zw6Hih6go8E7Xs=
 github.com/gaia-pipeline/protobuf v0.0.0-20180812091451-7be8a901b55a/go.mod h1:H0w7MofSuW53Nz7kesnBdVkvr437flf5B7D9Lcsb+lQ=
 github.com/gliderlabs/ssh v0.1.1 h1:j3L6gSLQalDETeEg/Jg0mGY0/y/N6zI2xX1978P0Uqw=

workers/agent/agent.go

@@ -23,6 +23,7 @@ import (
 	"github.com/gaia-pipeline/gaia/helper/pipelinehelper"
 	"github.com/gaia-pipeline/gaia/store"
 	"github.com/gaia-pipeline/gaia/workers/agent/api"
+	gp "github.com/gaia-pipeline/gaia/workers/pipeline"
 	pb "github.com/gaia-pipeline/gaia/workers/proto"
 	"github.com/gaia-pipeline/gaia/workers/scheduler"
 	"google.golang.org/grpc"
@@ -454,9 +455,76 @@ func (a *Agent) scheduleWork() {
 
 		// Let us try to start the plugin and receive all implemented jobs
 		if err = a.scheduler.SetPipelineJobs(pipeline); err != nil {
-			gaia.Cfg.Logger.Error("cannot get pipeline jobs", "error", err.Error(), "pipelinerun", pipelineRunPB)
-			reschedulePipeline()
-			return
+			if strings.Contains(err.Error(), "exec format error") {
+				gaia.Cfg.Logger.Info("pipeline in a different format than worker; attempting to rebuild...")
+				err = nil
+				// Try rebuilding the pipeline...
+				if err := os.Remove(pipelineFullPath); err != nil {
+					gaia.Cfg.Logger.Error("failed to remove pipeline binary", "error", err.Error(), "pipelinerun", pipelineRunPB)
+					reschedulePipeline()
+					return
+				}
+				pCreate := &gaia.CreatePipeline{}
+				repo, err := a.client.GetGitRepo(ctx, &pb.PipelineID{Id: int64(pipeline.ID)})
+				if err != nil {
+					gaia.Cfg.Logger.Error("failed to get pipeline information", "error", err.Error(), "pipelinerun", pipelineRunPB)
+					reschedulePipeline()
+					return
+				}
+				pCreate.Pipeline = *pipeline
+				// Unfortunately, since pb.GitRepo has extra gRPC fields on it
+				// we can't use gaia.GitRepo(repo) here to convert immediately.
+				gitRepo := gaia.GitRepo{}
+				gitRepo.Username = repo.Username
+				gitRepo.Password = repo.Password
+				pk := gaia.PrivateKey{}
+				pk.Password = repo.PrivateKey.Password
+				pk.Username = repo.PrivateKey.Username
+				pk.Key = repo.PrivateKey.Key
+				gitRepo.PrivateKey = pk
+				gitRepo.URL = repo.Url
+				gitRepo.SelectedBranch = repo.SelectedBranch
+				pCreate.Pipeline.Repo = &gitRepo
+				//gp.CreatePipeline(pCreate)
+				//if pCreate.StatusType == gaia.CreatePipelineFailed {
+				//	gaia.Cfg.Logger.Error("cannot create pipeline", "output", pCreate.Output, "pipelinerun", pipelineRunPB)
+				//	reschedulePipeline()
+				//	return
+				//}
+
+				pb := gp.NewBuildPipeline(pipeline.Type)
+				if err = pb.PrepareEnvironment(pCreate); err != nil {
+					gaia.Cfg.Logger.Error("cannot prepare pipeline environment by worker", "error", err.Error(), "pipelinerun", pipelineRunPB)
+					reschedulePipeline()
+					return
+				}
+				if err = gp.GitCloneRepo(pCreate.Pipeline.Repo); err != nil {
+					gaia.Cfg.Logger.Error("error pulling repo", "error", err.Error(), "pipelinerun", pipelineRunPB)
+					reschedulePipeline()
+					return
+				}
+
+				if err = pb.ExecuteBuild(pCreate); err != nil {
+					gaia.Cfg.Logger.Error("cannot execute build by worker", "error", err.Error(), "pipelinerun", pipelineRunPB)
+					reschedulePipeline()
+					return
+				}
+				if err = pb.CopyBinary(pCreate); err != nil {
+					gaia.Cfg.Logger.Error("cannot copy binary by worker", "error", err.Error(), "pipelinerun", pipelineRunPB)
+					reschedulePipeline()
+					return
+				}
+				pipeline = &pCreate.Pipeline
+				if err = a.scheduler.SetPipelineJobs(pipeline); err != nil {
+					gaia.Cfg.Logger.Error("cannot execute build by worker", "error", err.Error(), "pipelinerun", pipelineRunPB)
+					reschedulePipeline()
+					return
+				}
+			} else {
+				gaia.Cfg.Logger.Error("cannot get pipeline jobs", "error", err.Error(), "pipelinerun", pipelineRunPB)
+				reschedulePipeline()
+				return
+			}
 		}
 		pipelineRun.Jobs = pipeline.Jobs
 

workers/agent/agent_test.go

@@ -102,6 +102,10 @@ type mockWorkerInterface struct {
 	pbRuns []*pb.PipelineRun
 }
 
+func (mw *mockWorkerInterface) GetGitRepo(context.Context, *pb.PipelineID) (*pb.GitRepo, error) {
+	return &pb.GitRepo{}, nil
+}
+
 var mW *mockWorkerInterface
 
 func (mw *mockWorkerInterface) GetWork(workInst *pb.WorkerInstance, serv pb.Worker_GetWorkServer) error {

workers/pipeline/create_pipeline.go

@@ -44,7 +44,7 @@ func CreatePipeline(p *gaia.CreatePipeline) {
 	p.GitHubToken = ""
 	storeService, _ := services.StorageService()
 	// Define build process for the given type
-	bP := newBuildPipeline(p.Pipeline.Type)
+	bP := NewBuildPipeline(p.Pipeline.Type)
 	if bP == nil {
 		// Pipeline type is not supported
 		p.StatusType = gaia.CreatePipelineFailed
@@ -63,7 +63,7 @@ func CreatePipeline(p *gaia.CreatePipeline) {
 	}
 
 	// Clone git repo
-	err = gitCloneRepo(p.Pipeline.Repo)
+	err = GitCloneRepo(p.Pipeline.Repo)
 	if err != nil {
 		p.StatusType = gaia.CreatePipelineFailed
 		p.Output = fmt.Sprintf("cannot prepare build: %s", err.Error())

workers/pipeline/git.go

@@ -138,7 +138,7 @@ func UpdateRepository(pipe *gaia.Pipeline) error {
 	}
 
 	gaia.Cfg.Logger.Debug("updating pipeline: ", "message", pipe.Name)
-	b := newBuildPipeline(pipe.Type)
+	b := NewBuildPipeline(pipe.Type)
 	createPipeline := &gaia.CreatePipeline{
 		Pipeline: gaia.Pipeline{
 			Repo: &gaia.GitRepo{},
@@ -151,9 +151,9 @@ func UpdateRepository(pipe *gaia.Pipeline) error {
 	return nil
 }
 
-// gitCloneRepo clones the given repo to a local folder.
+// GitCloneRepo clones the given repo to a local folder.
 // The destination will be attached to the given repo obj.
-func gitCloneRepo(repo *gaia.GitRepo) error {
+func GitCloneRepo(repo *gaia.GitRepo) error {
 	// Check if credentials were provided
 	auth, err := getAuthInfo(repo, nil)
 	if err != nil {

workers/pipeline/git_test.go

@@ -23,7 +23,7 @@ func TestGitCloneRepo(t *testing.T) {
 		URL:       "https://github.com/gaia-pipeline/pipeline-test",
 		LocalDest: tmp,
 	}
-	err := gitCloneRepo(repo)
+	err := GitCloneRepo(repo)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -69,7 +69,7 @@ func TestUpdateAllPipelinesAlreadyUpToDate(t *testing.T) {
 	}
 	// always ensure that tmp folder is cleaned up
 	defer os.RemoveAll("tmp")
-	err := gitCloneRepo(repo)
+	err := GitCloneRepo(repo)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -125,7 +125,7 @@ Xbs5AQIEIzWnmQIFAOEml+E=
 
 	// always ensure that tmp folder is cleaned up
 	defer os.RemoveAll("tmp")
-	_ = gitCloneRepo(repo)
+	_ = GitCloneRepo(repo)
 	want := "knownhosts: key is unknown"
 	if !strings.Contains(b.String(), want) {
 		t.Fatalf("wanted buf to contain: '%s', got: '%s'", want, b.String())
@@ -170,7 +170,7 @@ Xbs5AQIEIzWnmQIFAOEml+E=
 
 	// always ensure that tmp folder is cleaned up
 	defer os.RemoveAll("tmp")
-	_ = gitCloneRepo(repo)
+	_ = GitCloneRepo(repo)
 
 	p := new(gaia.Pipeline)
 	p.Name = "main"
@@ -209,7 +209,7 @@ func TestUpdateAllPipelinesAlreadyUpToDateWithMoreThanOnePipeline(t *testing.T)
 	}
 	// always ensure that tmp folder is cleaned up
 	defer os.RemoveAll("tmp")
-	err := gitCloneRepo(repo)
+	err := GitCloneRepo(repo)
 	if err != nil {
 		t.Fatal(err)
 	}

workers/pipeline/pipeline.go

@@ -66,9 +66,9 @@ var execCommandContext = exec.CommandContext
 // Source folder name where the sources are stored
 const srcFolder = "src"
 
-// newBuildPipeline creates a new build pipeline for the given
+// NewBuildPipeline creates a new build pipeline for the given
 // pipeline type.
-func newBuildPipeline(t gaia.PipelineType) BuildPipeline {
+func NewBuildPipeline(t gaia.PipelineType) BuildPipeline {
 	var bP BuildPipeline
 
 	// Create build pipeline for given pipeline type

workers/pipeline/pipeline_test.go

@@ -347,27 +347,27 @@ func TestGetExecPath(t *testing.T) {
 }
 
 func TestNewBuildPipeline(t *testing.T) {
-	goBuildPipeline := newBuildPipeline(gaia.PTypeGolang)
+	goBuildPipeline := NewBuildPipeline(gaia.PTypeGolang)
 	if goBuildPipeline == nil {
 		t.Fatalf("should be of type %s but is nil\n", gaia.PTypeGolang)
 	}
-	javaBuildPipeline := newBuildPipeline(gaia.PTypeJava)
+	javaBuildPipeline := NewBuildPipeline(gaia.PTypeJava)
 	if javaBuildPipeline == nil {
 		t.Fatalf("should be of type %s but is nil\n", gaia.PTypeJava)
 	}
-	pythonBuildPipeline := newBuildPipeline(gaia.PTypePython)
+	pythonBuildPipeline := NewBuildPipeline(gaia.PTypePython)
 	if pythonBuildPipeline == nil {
 		t.Fatalf("should be of type %s but is nil\n", gaia.PTypePython)
 	}
-	cppBuildPipeline := newBuildPipeline(gaia.PTypeCpp)
+	cppBuildPipeline := NewBuildPipeline(gaia.PTypeCpp)
 	if cppBuildPipeline == nil {
 		t.Fatalf("should be of type %s but is nil\n", gaia.PTypeCpp)
 	}
-	rubyBuildPipeline := newBuildPipeline(gaia.PTypeRuby)
+	rubyBuildPipeline := NewBuildPipeline(gaia.PTypeRuby)
 	if rubyBuildPipeline == nil {
 		t.Fatalf("should be of type %s but is nil\n", gaia.PTypeRuby)
 	}
-	nodeJSBuildPipeline := newBuildPipeline(gaia.PTypeNodeJS)
+	nodeJSBuildPipeline := NewBuildPipeline(gaia.PTypeNodeJS)
 	if nodeJSBuildPipeline == nil {
 		t.Fatalf("should be of type %s but is nil\n", gaia.PTypeNodeJS)
 	}