feat wait on quotqa exceeded so quotqa recovers (#153)

Author: BrunoReboul

services/getgroupsettings/core.go

@@ -39,6 +39,8 @@ import (
 	pubsubpb "google.golang.org/genproto/googleapis/pubsub/v1"
 )
 
+const waitSecOnQuotaExceeded = 70
+
 // Global structure for global variables to optimize the cloud function performances
 type Global struct {
 	ctx                   context.Context
@@ -256,6 +258,19 @@ func EntryPoint(ctxEvent context.Context, PubSubMessage gps.PubSubMessage, globa
 	if !feedMessageGroup.Deleted {
 		groupSettings, err := global.groupsSettingsService.Groups.Get(feedMessageGroup.Asset.Resource.Email).Do()
 		if err != nil {
+			if strings.Contains(strings.ToLower(err.Error()), "quota") {
+				log.Println(glo.Entry{
+					MicroserviceName:   global.microserviceName,
+					InstanceName:       global.instanceName,
+					Environment:        global.environment,
+					Severity:           "WARNING",
+					Message:            fmt.Sprintf("waiting_on_quota_exceeded"),
+					Description:        fmt.Sprintf("GetGroupSettings quota is gone, wait for %d seconds then retry", waitSecOnQuotaExceeded),
+					TriggeringPubsubID: global.PubSubID,
+				})
+				time.Sleep(waitSecOnQuotaExceeded * time.Second)
+				return err
+			}
 			log.Println(glo.Entry{
 				MicroserviceName:   global.microserviceName,
 				InstanceName:       global.instanceName,

services/listgroupmembers/core.go

@@ -40,6 +40,8 @@ import (
 	admin "google.golang.org/api/admin/directory/v1"
 )
 
+const waitSecOnQuotaExceeded = 70
+
 // Global variable to deal with GroupsListCall Pages constraint: no possible to pass variable to the function in pages()
 // https://pkg.go.dev/google.golang.org/api/admin/directory/v1?tab=doc#GroupsListCall.Pages
 var ancestors []string
@@ -311,6 +313,19 @@ func EntryPoint(ctxEvent context.Context, PubSubMessage gps.PubSubMessage, globa
 		// pages function except just the name of the callback function. Not an invocation of the function
 		err = global.dirAdminService.Members.List(feedMessageGroup.Asset.Resource.Id).MaxResults(global.maxResultsPerPage).Pages(ctx, browseMembers)
 		if err != nil {
+			if strings.Contains(strings.ToLower(err.Error()), "quota") {
+				log.Println(glo.Entry{
+					MicroserviceName:   global.microserviceName,
+					InstanceName:       global.instanceName,
+					Environment:        global.environment,
+					Severity:           "WARNING",
+					Message:            fmt.Sprintf("waiting_on_quota_exceeded"),
+					Description:        fmt.Sprintf("ListGroupMembers quota is gone, wait for %d seconds then retry", waitSecOnQuotaExceeded),
+					TriggeringPubsubID: global.PubSubID,
+				})
+				time.Sleep(waitSecOnQuotaExceeded * time.Second)
+				return err
+			}
 			log.Println(glo.Entry{
 				MicroserviceName:   global.microserviceName,
 				InstanceName:       global.instanceName,

services/listgroups/core.go

@@ -39,6 +39,8 @@ import (
 	admin "google.golang.org/api/admin/directory/v1"
 )
 
+const waitSecOnQuotaExceeded = 70
+
 // Global variable to deal with GroupsListCall Pages constraint: no possible to pass variable to the function in pages()
 // https://pkg.go.dev/google.golang.org/api/admin/directory/v1?tab=doc#GroupsListCall.Pages
 var ctx context.Context
@@ -340,6 +342,19 @@ func EntryPoint(ctxEvent context.Context, PubSubMessage gps.PubSubMessage, globa
 
 			err = queryDirectory(settings.Domain, settings.EmailPrefix, global)
 			if err != nil {
+				if strings.Contains(strings.ToLower(err.Error()), "quota") {
+					log.Println(glo.Entry{
+						MicroserviceName:   global.microserviceName,
+						InstanceName:       global.instanceName,
+						Environment:        global.environment,
+						Severity:           "WARNING",
+						Message:            fmt.Sprintf("waiting_on_quota_exceeded"),
+						Description:        fmt.Sprintf("ListGroup quota is gone, wait for %d seconds then retry", waitSecOnQuotaExceeded),
+						TriggeringPubsubID: global.PubSubID,
+					})
+					time.Sleep(waitSecOnQuotaExceeded * time.Second)
+					return err
+				}
 				log.Println(glo.Entry{
 					MicroserviceName:   global.microserviceName,
 					InstanceName:       global.instanceName,