cmd/collector/collector.go

@@ -31,36 +31,41 @@ import (
 	"collector"
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
-	"net"
 	"net/http"
 	"os"
 
 	"clouditor.io/clouditor/api/discovery"
-	"clouditor.io/clouditor/logging/formatter"
-	"clouditor.io/clouditor/rest"
-	"clouditor.io/clouditor/service"
+	"clouditor.io/clouditor/persistence"
+	"clouditor.io/clouditor/persistence/gorm"
+	server_clouditor "clouditor.io/clouditor/server"
+	"clouditor.io/clouditor/server/rest"
 	service_discovery "clouditor.io/clouditor/service/discovery"
 	"clouditor.io/clouditor/voc"
+
 	structpb "github.com/golang/protobuf/ptypes/struct"
-	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
-	grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth"
-	grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
-	grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/oauth2/clientcredentials"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/reflection"
 )
 
 var (
 	log              *logrus.Entry
 	server           *grpc.Server
 	discoveryService discovery.DiscoveryServer
-	assessmentUrl    string
-	jwksURL          string
-	config           clientcredentials.Config
+	storage          persistence.Storage
+
+	assessmentUrl   string
+	jwksURL         string
+	config          clientcredentials.Config
+	cloudServiceID  string
+	resourceGroupId string
+	dbHost          string
+	dbUser          string
+	dbPassword      string
+
+	// provider sets the cloud service provider an evidence collector instance (pod) should gather resources from
+	provider string
 )
 
 const (
@@ -72,6 +77,12 @@ const (
 	OAuth2ClientSecret   = "OAUTH2_CLIENT_SECRET"
 	OAuth2TokenURL       = "OAUTH2_TOKEN_URL"
 	JWKSURL              = "AUTH_JWKS_URL"
+	CloudServiceID       = "CLOUD_SERVICE_ID"
+	ResourceGroupId      = "RESOURCE_GROUP_ID"
+	PostgresHost         = "POSTGRES_HOST"
+	PostgresUser         = "POSTGRES_USER"
+	PostgresPassword     = "POSTGRES_PASSWORD"
+	Provider             = "PROVIDER"
 )
 
 func toStruct(r voc.IsCloudResource) (s *structpb.Value) {
@@ -111,15 +122,33 @@ func init() {
 	// Get the JWKS URL of our auth server
 	jwksURL, _ = os.LookupEnv(JWKSURL)
 
+	cloudServiceID, ok = os.LookupEnv(CloudServiceID)
+	if !ok {
+		cloudServiceID = discovery.DefaultCloudServiceID
+	}
+
+	resourceGroupId, ok = os.LookupEnv(ResourceGroupId)
+	if !ok {
+		resourceGroupId = ""
+	}
+
 	// Get the OAuth credentials
 	config = clientcredentials.Config{}
 	config.ClientID, _ = os.LookupEnv(OAuth2ClientID)
 	config.ClientSecret, _ = os.LookupEnv(OAuth2ClientSecret)
 	config.TokenURL, _ = os.LookupEnv(OAuth2TokenURL)
+
+	// Get DB credentials
+	dbHost, _ = os.LookupEnv(PostgresHost)
+	dbUser, _ = os.LookupEnv(PostgresUser)
+	dbPassword, _ = os.LookupEnv(PostgresPassword)
 }
 
 func main() {
-	var textFormatter = logrus.TextFormatter{ForceColors: false, FullTimestamp: true}
+	var (
+		textFormatter = logrus.TextFormatter{ForceColors: false, FullTimestamp: true}
+		err           error
+	)
 	log.Logger.Formatter = &textFormatter
 
 	fmt.Printf(`
@@ -137,66 +166,72 @@ func main() {
 
 	log.Infof("Security Assessment URL is set to %s", assessmentUrl)
 
+	storage, err = gorm.NewStorage(gorm.WithPostgres(dbHost, 5432, dbUser, dbPassword, "postgres", ""))
+	if err != nil {
+		log.Errorf("Could not connect to storage (yet): %v", err)
+		os.Exit(1)
+	}
+
+	// Get provider (one is enough because 1 cloud service == 1 provider)
+	provider = os.Getenv(Provider)
+	if provider == "" {
+		log.Errorf("Exiting. No provider specified: %v", err)
+		os.Exit(1)
+	}
+
 	discoveryService = service_discovery.NewService(
 		service_discovery.WithOAuth2Authorizer(&config),
-		service_discovery.WithProviders([]string{service_discovery.ProviderAzure}),
+		service_discovery.WithProviders([]string{provider}),
 		service_discovery.WithAssessmentAddress(assessmentUrl),
+		service_discovery.WithCloudServiceID(cloudServiceID),
+		service_discovery.WithStorage(storage),
 	)
 
 	// Comment in if evidence collector should collect the evidences from the CSPs. Otherwise, example evidences are used.
-	//Start evidence collector
-	_, err := discoveryService.Start(context.Background(), &discovery.StartDiscoveryRequest{})
+	// Start evidence collector (possibly scoped to the resourcegroup)
+	if resourceGroupId == "" {
+		_, err = discoveryService.Start(context.Background(), &discovery.StartDiscoveryRequest{})
+	} else {
+		_, err = discoveryService.Start(context.Background(), &discovery.StartDiscoveryRequest{
+			ResourceGroup: &resourceGroupId,
+		})
+	}
+
 	if err != nil {
 		log.Errorf("could not collect evidences: %v", err)
 	}
 
-	grpcLogger := logrus.New()
-	grpcLogger.Formatter = &formatter.GRPCFormatter{TextFormatter: textFormatter}
-	grpcLoggerEntry := grpcLogger.WithField("component", "grpc")
+	log.Infof("Starting gRPC endpoint on :%d", grpcPort)
 
-	// create a new socket for gRPC communication
-	sock, err := net.Listen("tcp", fmt.Sprintf(":%d", grpcPort))
+	// Add grpc opts
+	grpcOpts := []grpc.ServerOption{
+		// Add max grpc message sizes
+		grpc.MaxRecvMsgSize(1024 * 1024 * 20),
+		grpc.MaxSendMsgSize(1024 * 1024 * 20)}
+
+	// Start the gRPC server
+	_, server, err = server_clouditor.StartGRPCServer(
+		fmt.Sprintf("0.0.0.0:%d", grpcPort),
+		server_clouditor.WithJWKS(jwksURL),
+		server_clouditor.WithDiscovery(discoveryService),
+		server_clouditor.WithReflection(),
+		server_clouditor.WithAdditionalGRPCOpts(grpcOpts),
+	)
 	if err != nil {
-		log.Errorf("could not listen: %v", err)
+		log.Errorf("Failed to serve gRPC endpoint: %s", err)
+		return
 	}
 
-	authConfig := service.ConfigureAuth(service.WithJWKSURL(jwksURL))
-	defer authConfig.Jwks.EndBackground()
-
-	server = grpc.NewServer(
-		grpc_middleware.WithUnaryServerChain(
-			grpc_ctxtags.UnaryServerInterceptor(grpc_ctxtags.WithFieldExtractor(grpc_ctxtags.CodeGenRequestFieldExtractor)),
-			grpc_logrus.UnaryServerInterceptor(grpcLoggerEntry),
-			grpc_auth.UnaryServerInterceptor(authConfig.AuthFunc),
-		),
-		grpc_middleware.WithStreamServerChain(
-			grpc_ctxtags.StreamServerInterceptor(grpc_ctxtags.WithFieldExtractor(grpc_ctxtags.CodeGenRequestFieldExtractor)),
-			grpc_logrus.StreamServerInterceptor(grpcLoggerEntry),
-			grpc_auth.StreamServerInterceptor(authConfig.AuthFunc),
-		))
-
-	discovery.RegisterDiscoveryServer(server, discoveryService)
-	// enable reflection, primary for testing in early stages
-	reflection.Register(server)
-
-	// start the gRPC-HTTP gateway
-	go func() {
-		err = rest.RunServer(context.Background(), grpcPort, httpPort)
-		if errors.Is(err, http.ErrServerClosed) {
-			os.Exit(0)
-			return
-		}
-
-		if err != nil {
-			log.Fatalf("failed to serve gRPC-HTTP gateway: %v", err)
-		}
-	}()
-
-	log.Infof("Starting gRPC endpoint on :%d", grpcPort)
-
-	// serve the gRPC socket
-	if err := server.Serve(sock); err != nil {
-		log.Infof("failed to serve gRPC endpoint: %s", err)
+	// Start the gRPC-HTTP gateway
+	err = rest.RunServer(context.Background(),
+		grpcPort,
+		httpPort,
+	)
+	if err != nil && err != http.ErrServerClosed {
+		log.Errorf("failed to serve gRPC-HTTP gateway: %v", err)
 		return
 	}
+
+	log.Infof("Stopping gRPC endpoint")
+	server.GracefulStop()
 }

go.mod

 module collector
 
-go 1.18
+go 1.20
 
 require (
-	clouditor.io/clouditor v1.4.15
-	github.com/golang/protobuf v1.5.2
-	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
-	github.com/sirupsen/logrus v1.8.1
-	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
-	google.golang.org/grpc v1.46.0
+	clouditor.io/clouditor v1.9.4-0.20230726134626-03c5b3f7af29
+	github.com/golang/protobuf v1.5.3
+	github.com/sirupsen/logrus v1.9.3
+	golang.org/x/oauth2 v0.8.0
+	google.golang.org/grpc v1.57.0-dev.0.20230612212144-642dd63a8527
 )
 
 require (
-	cloud.google.com/go v0.99.0 // indirect
-	github.com/Azure/azure-sdk-for-go v64.1.0+incompatible // indirect
-	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.24 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
-	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 // indirect
-	github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect
-	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
-	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
-	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
-	github.com/Azure/go-autorest/logger v0.2.1 // indirect
-	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/MicahParks/keyfunc v1.1.0 // indirect
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
-	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.16.3 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.0 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.15.0 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.10.0 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ec2 v1.43.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/lambda v1.23.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.26.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.11.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.16.0 // indirect
-	github.com/aws/smithy-go v1.11.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0-beta.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.1.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v3 v3.0.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dataprotection/armdataprotection v1.0.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/security/armsecurity v0.11.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/sql/armsql v1.1.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription v1.1.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
+	github.com/MicahParks/keyfunc/v2 v2.1.0 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.19.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.18.24 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.23 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.104.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/lambda v1.37.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 // indirect
+	github.com/aws/smithy-go v1.13.5 // indirect
+	github.com/bmatcuk/doublestar/v4 v4.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dimchansky/utfbom v1.1.1 // indirect
-	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
-	github.com/go-co-op/gocron v1.13.0 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
-	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.19.5 // indirect
-	github.com/go-openapi/swag v0.19.14 // indirect
-	github.com/gofrs/uuid v4.1.0+incompatible // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.0.0 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/glebarez/go-sqlite v1.21.1 // indirect
+	github.com/glebarez/sqlite v1.8.0 // indirect
+	github.com/go-co-op/gocron v1.28.0 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
-	github.com/google/gnostic v0.6.8 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
+	github.com/golang/glog v1.1.1 // indirect
+	github.com/google/addlicense v1.1.0 // indirect
+	github.com/google/gnostic v0.6.9 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
+	github.com/iancoleman/strcase v0.2.0 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgx/v5 v5.3.1 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/logrusorgru/aurora/v3 v3.0.0 // indirect
-	github.com/mailru/easyjson v0.7.6 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/lyft/protoc-gen-star v0.6.1 // indirect
+	github.com/lyft/protoc-gen-star/v2 v2.0.3 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-isatty v0.0.18 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/oxisto/oauth2go v0.5.12 // indirect
+	github.com/oxisto/oauth2go v0.9.0 // indirect
+	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
+	github.com/spf13/afero v1.9.5 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/srikrsna/protoc-gen-gotag v0.6.2 // indirect
-	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
-	golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd // indirect
-	golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	golang.org/x/crypto v0.11.0 // indirect
+	golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect
+	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
+	golang.org/x/mod v0.10.0 // indirect
+	golang.org/x/net v0.11.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/term v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.9.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/api v0.24.0 // indirect
-	k8s.io/apimachinery v0.24.0 // indirect
-	k8s.io/client-go v0.24.0 // indirect
-	k8s.io/klog/v2 v2.60.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect
-	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
-	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gorm.io/driver/postgres v1.5.0 // indirect
+	gorm.io/gorm v1.25.1 // indirect
+	k8s.io/api v0.27.1 // indirect
+	k8s.io/apimachinery v0.27.1 // indirect
+	k8s.io/client-go v0.27.1 // indirect
+	k8s.io/klog/v2 v2.100.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
+	k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect
+	modernc.org/libc v1.22.5 // indirect
+	modernc.org/mathutil v1.5.0 // indirect
+	modernc.org/memory v1.5.0 // indirect
+	modernc.org/sqlite v1.22.1 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )